diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5294ab1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# vim files +.*~ +.*.s?? + +# Ignore all maven build and test files +target/* +*.class + +# Eclipse +.classpath +.project +.settings/ + +# Ignore all IntelliJ IDEA files +.idea/* +*.iml +*.iws +.attach_* + +# Ignore all OSX hidden files +*.DS_Store +/bin/ + +# ignore submission files +hw*.zip + diff --git a/README.md b/README.md new file mode 100644 index 0000000..dc56657 --- /dev/null +++ b/README.md @@ -0,0 +1,422 @@ +# MOOCbase + +This repo contains a bare-bones database implementation, which supports +executing simple transactions in series. In the homeworks of +this class, you will be adding to this implementation, adding support for +B+ tree indices, efficient join algorithms, query optimization, multigranularity +locking to support concurrent execution of transactions, and database recovery. + +As you will be working with this codebase for the rest of the semester, it is +a good idea to get familiar with it. + +## Overview + +In this document, we explain + +- how to fetch the released code from GitHub +- how to fetch any updates to the released code +- how to setup a local development environment +- how to run tests inside the CS186 docker container and using IntelliJ +- how to submit your code to turn in homeworks +- how to reset your docker container +- the general architecture of the released code + +## Fetching the released code + +The **first** time you work on this codebase, run the following command (this is +the same command given in HW0; you do not need to run this again if you are +working through HW0): + +```bash +docker run --name cs186 -v "/path/to/cs186/directory/on/your/machine:/cs186" -it cs186/environment /bin/bash +``` +This should start a bash shell (type `exit` to exit). You should only need to run this +one time in this class. + +To start the container, run: +```bash +docker start -ai cs186 +``` + +After some notifications, you should get a prompt like this: +``` +ubuntu@1891ee9ee645:/$ +``` +This indicates that you are inside the container; to exit the container, type +`exit` or `ctrl+D` at the bash prompt. + +While inside the container, navigate to the shared directory: +```bash +cd /cs186 +``` + +Clone this repo: +```bash +git clone https://github.com/berkeley-cs186/sp20-moocbase.git +``` + +If you get an error like +`Could not resolve host: github.com`, try restarting your docker machine (exit the +container and run `docker-machine restart`) or restarting your computer. + +Now, navigate into the newly created directory: +```bash +cd sp20-moocbase +``` + +To test that everything is working correctly, run: +```bash +mvn clean test -P system +``` + +There should not be any failures. + +## Fetching any updates to the released code + +In a perfect world, we would never have to update the released code, because +it would be perfectly free of bugs. Unfortunately, bugs do surface from time to +time, and you may have to fetch updates. We will post on Piazza whenever +fetching updates is necessary. The following instructions explain how to do so. + +Inside the container, navigate to the cloned repo: +```bash +cd /cs186/sp20-moocbase +``` + +Commit any changes you have made so far: +```bash +git add --all . +git commit -m "commit message" +``` + +If you get the following error: +``` +*** Please tell me who you are. + +Run + + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + +to set your account's default identity. +Omit --global to set the identity only in this repository. + +fatal: empty ident name (for ) not allowed +``` +then run the two suggested commands and try again. + +Pull the changes: +```bash +git pull --rebase +``` + +If there are any conflicts, resolve them and run `git rebase --continue`. If you +need help resolving merge conflicts, please come to office hours. + +## Setting up your local development environment + +You are free to use any text editor or IDE to complete the homeworks, but **we +will build and test your code in the docker container with maven**. + +We recommend setting up a more local development environment by installing Java +8 locally (the version our Docker container runs) and using an IDE such as +IntelliJ. + +[Java 8 downloads](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) + +If you have another version of Java installed, it's probably fine to use it, as +long as you do not use any features not in Java 8. You should run tests +somewhat frequently inside the container to make sure that your code works with +our setup. + +To import the project into IntelliJ, make sure that you import as a maven +project (select the pom.xml file when importing). Make sure that you can compile +your code and run tests (it's ok if there are a lot of failed tests - you +haven't begun implementing anything yet!). You should also make sure that you +can run the debugger and step through code. + +## Running tests + +The code in this repository comes with a set of tests that can be run. These +test cases are not comprehensive, and we suggest writing your own to ensure that +your code is correct. + +To run all the tests for a particular homework, start the container, navigate +to the cloned repo (`cd /cs186/sp20-moocbase`), and run the following command: +```bash +mvn clean test -DHW=n +``` + +but replace `n` with one of: `0`, `2`, `3Part1`, `3Part2`, `4Part1`, `4Part2`, `5`. + +Before submitting any homeworks, be sure to run tests inside your container. **We +will not be accepting "the test ran successfully in my IDE" as an excuse -- you +are responsible for making sure the tests run successfully _in the docker +container_**. + +A (very) common problem that students in past semester ran into when they did +not run tests in Docker before submitting was encountering an error similar to +the following after submitting: + +``` +[INFO] ----------------------------------------------------------- +[ERROR] COMPILATION ERROR +[INFO] ----------------------------------------------------------- +[ERROR] /cs186/sp20-moocbase/src/main/java/edu/berkeley/cs186/database/index/BPlusTr +ee.java:[3, 45] package com.sun.internal.rngom.parse.host does not exist +[INFO] 1 error +[INFO] ----------------------------------------------------------- +[INFO] ----------------------------------------------------------------------- +[INFO] BUILD FAILURE +[INFO] ----------------------------------------------------------------------- +``` + +When your IDE lists autocomplete options, there are sometimes classes/enums/etc. +from non-standard libraries. If you select one by accident, your IDE may +automatically add the import for it, which stays even after you delete the +incorrectly autocompleted word. + +As the import is (probably) unused, just go to the specified file and line and +delete the import. + +We will be running tests with the same Docker image as you, so running tests in +Docker before submitting lets you check that everything will work as intended +when we run tests to grade your submission. + +### Running tests in IntelliJ + +If you are using IntelliJ, and wish to run the same tests that +`mvn clean test -DHW=n` runs, follow the instructions in the following document: + +[IntelliJ setup](intellij-test-setup.md) + +## Submitting homeworks + +To submit a homework, **start your container**, navigate to the cloned repo, and +run: +```bash +python3 turn_in.py +``` + +This will generate a zip file. Upload the zip file to the Homework Submission +assignment on edX. + +Note that you are only allowed to modify certain files for each homework, and +changes to other files you are not allowed to modify will be discarded when we +run tests. + +## Resetting the Docker container + +If things are not working in the Docker container, a first step for +troubleshooting is to start a container from the image again. This will discard +any changes made in the container's filesystem, but will not discard changes +made inside a mounted folder (i.e. `/cs186`). + +Outside of Docker, first delete the container: + +```bash +docker container rm cs186 +``` + +Then, create it again (this is the command you ran back in HW0): + +```bash +docker run --name cs186 -v ":/cs186" -it cs186/environment /bin/bash +``` + +## The code + +The code is located in the `src/main/java/edu/berkeley/cs186/database` +directory, while the tests are located in the +`src/test/java/edu/berkeley/cs186/database` directory. + +### common + +The `common` directory contains bits of useful code and general interfaces that +are not limited to any one part of the codebase. + +### concurrency + +The `concurrency` directory contains a skeleton for adding multigranularity +locking to the database. You will be implementing this in HW4. + +### databox + +Our database has, like most DBMS's, a type system distinct from that of the +programming language used to implement the DBMS. (Our DBMS doesn't quite provide +SQL types either, but it's modeled on a simplified version of SQL types). + +The `databox` directory contains classes which represents values stored in +a database, as well as their types. The various `DataBox` classes represent +values of certain types, whereas the `Type` class represents types used in the +database. + +An example: +```java +DataBox x = new IntDataBox(42); // The integer value '42'. +Type t = Type.intType(); // The type 'int'. +Type xsType = x.type(); // Get x's type, which is Type.intType(). +int y = x.getInt(); // Get x's value: 42. +String s = x.getString(); // An exception is thrown, since x is not a string. +``` + +### index + +The `index` directory contains a skeleton for implementing B+ tree indices. You +will be implementing this in HW2. + +### memory + +The `memory` directory contains classes for managing the loading of data +into and out of memory (in other words, buffer management). + +The `BufferFrame` class represents a single buffer frame (page in the buffer +pool) and supports pinning/unpinning and reading/writing to the buffer frame. +All reads and writes require the frame be pinned (which is often done via the +`requireValidFrame` method, which reloads data from disk if necessary, and then +returns a pinned frame for the page). + +The `BufferManager` interface is the public interface for the buffer manager of +our DBMS. + +The `BufferManagerImpl` class implements a buffer manager using +a write-back buffer cache with configurable eviction policy. It is responsible +for fetching pages (via the disk space manager) into buffer frames, and returns +Page objects to allow for manipulation of data in memory. + +The `Page` class represents a single page. When data in the page is accessed or +modified, it delegates reads/writes to the underlying buffer frame containing +the page. + +The `EvictionPolicy` interface defines a few methods that determine how the +buffer manager evicts pages from memory when necessary. Implementations of these +include the `LRUEvictionPolicy` (for LRU) and `ClockEvictionPolicy` (for clock). + +### io + +The `io` directory contains classes for managing data on-disk (in other words, +disk space management). + +The `DiskSpaceManager` interface is the public interface for the disk space +manager of our DBMS. + +The `DiskSpaceMangerImpl` class is the implementation of the disk space +manager, which maps groups of pages (partitions) to OS-level files, assigns +each page a virtual page number, and loads/writes these pages from/to disk. + +### query + +The `query` directory contains classes for managing and manipulating queries. + +The various operator classes are query operators (pieces of a query), some of +which you will be implementing in HW3. + +The `QueryPlan` class represents a plan for executing a query (which we will be +covering in more detail later in the semester). It currently executes the query +as given (runs things in logical order, and performs joins in the order given), +but you will be implementing +a query optimizer in HW3 to run the query in a more efficient manner. + +### recovery + +The `recovery` directory contains a skeleton for implementing database recovery +a la ARIES. You will be implementing this in HW5. + +### table + +The `table` directory contains classes representing entire tables and records. + +The `Table` class is, as the name suggests, a table in our database. See the +comments at the top of this class for information on how table data is layed out +on pages. + +The `Schema` class represents the _schema_ of a table (a list of column names +and their types). + +The `Record` class represents a record of a table (a single row). Records are +made up of multiple DataBoxes (one for each column of the table it belongs to). + +The `RecordId` class identifies a single record in a table. + +The `HeapFile` interface is the interface for a heap file that the `Table` class +uses to find pages to write data to. + +The `PageDirectory` class is an implementation of `HeapFile` that uses a page +directory. + +#### table/stats + +The `table/stats` directory contains classes for keeping track of statistics of +a table. These are used to compare the costs of different query plans, when you +implement query optimization in HW4. + +### Transaction.java + +The `Transaction` interface is the _public_ interface of a transaction - it +contains methods that users of the database use to query and manipulate data. + +This interface is partially implemented by the `AbstractTransaction` abstract +class, and fully implemented in the `Database.Transaction` inner class. + +### TransactionContext.java + +The `TransactionContext` interface is the _internal_ interface of a transaction - +it contains methods tied to the current transaction that internal methods +(such as a table record fetch) may utilize. + +The current running transaction's transaction context is set at the beginning +of a `Database.Transaction` call (and available through the static +`getCurrentTransaction` method) and unset at the end of the call. + +This interface is partially implemented by the `AbstractTransactionContext` abstract +class, and fully implemented in the `Database.TransactionContext` inner class. + +### Database.java + +The `Database` class represents the entire database. It is the public interface +of our database - we do not parse SQL statements in our database, and instead, +users of our database use it like a Java library. + +All work is done in transactions, so to use the database, a user would start +a transaction with `Database#beginTransaction`, then call some of +`Transaction`'s numerous methods to perform selects, inserts, and updates. + +For example: +```java +Database db = new Database("database-dir"); + +try (Transaction t1 = db.beginTransaction()) { + Schema s = new Schema( + Arrays.asList("id", "firstName", "lastName"), + Arrays.asList(Type.intType(), Type.stringType(10), Type.stringType(10)) + ); + t1.createTable(s, "table1"); + t1.insert("table1", Arrays.asList( + new IntDataBox(1), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + )); + t1.insert("table1", Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + )); + t1.commit(); +} + +try (Transaction t2 = db.beginTransaction()) { + // .query("table1") is how you run "SELECT * FROM table1" + Iterator iter = t2.query("table1").execute(); + + System.out.println(iter.next()); // prints [1, John, Doe] + System.out.println(iter.next()); // prints [2, Jane, Doe] + + t2.commit(); +} + +db.close(); +``` + +More complex queries can be found in +[`src/test/java/edu/berkeley/cs186/database/TestDatabase.java`](src/test/java/edu/berkeley/cs186/database/TestDatabase.java). + diff --git a/courses.csv b/courses.csv new file mode 100644 index 0000000..2ced4a3 --- /dev/null +++ b/courses.csv @@ -0,0 +1,15 @@ +1,CS 186,Computer Science +2,CS 61A,Computer Science +3,BIO 1A,Biology +4,BIO 1B,Biology +5,CHEM 1A,Chemistry +6,CHEM 4A,Chemistry +7,UGBA 10,Business +8,UGBA 196,Business +9,ART 10,Art History +10,ART 100,Art History +11,ZOO 15,Zoology +12,ZOO 137B,Zoology +13,CS 162,Computer Science +14,BIO 100,Biology +15,CS 61C,Computer Science diff --git a/enrollments.csv b/enrollments.csv new file mode 100644 index 0000000..8c74446 --- /dev/null +++ b/enrollments.csv @@ -0,0 +1,1000 @@ +1,10 +1,6 +1,14 +1,8 +1,5 +2,11 +2,3 +2,1 +2,1 +2,15 +3,11 +3,7 +3,8 +3,1 +3,12 +4,5 +4,7 +4,8 +4,3 +4,9 +5,2 +5,12 +5,3 +5,7 +5,4 +6,4 +6,8 +6,9 +6,8 +6,6 +7,14 +7,7 +7,6 +7,11 +7,7 +8,2 +8,8 +8,2 +8,3 +8,7 +9,2 +9,13 +9,11 +9,2 +9,3 +10,3 +10,11 +10,11 +10,4 +10,13 +11,9 +11,9 +11,5 +11,8 +11,14 +12,1 +12,15 +12,3 +12,5 +12,10 +13,4 +13,15 +13,8 +13,12 +13,11 +14,5 +14,10 +14,3 +14,6 +14,14 +15,14 +15,15 +15,11 +15,11 +15,14 +16,9 +16,2 +16,7 +16,3 +16,7 +17,13 +17,9 +17,14 +17,14 +17,4 +18,8 +18,14 +18,4 +18,14 +18,9 +19,1 +19,5 +19,4 +19,1 +19,3 +20,15 +20,3 +20,3 +20,1 +20,7 +21,7 +21,10 +21,15 +21,3 +21,3 +22,13 +22,11 +22,11 +22,3 +22,6 +23,11 +23,12 +23,10 +23,15 +23,2 +24,11 +24,7 +24,5 +24,5 +24,9 +25,2 +25,5 +25,1 +25,12 +25,8 +26,4 +26,15 +26,14 +26,4 +26,5 +27,15 +27,14 +27,7 +27,11 +27,2 +28,13 +28,12 +28,1 +28,9 +28,14 +29,8 +29,11 +29,13 +29,1 +29,6 +30,6 +30,8 +30,2 +30,12 +30,1 +31,2 +31,10 +31,9 +31,13 +31,1 +32,9 +32,2 +32,2 +32,12 +32,10 +33,6 +33,9 +33,3 +33,14 +33,15 +34,4 +34,11 +34,5 +34,12 +34,2 +35,13 +35,14 +35,15 +35,6 +35,11 +36,2 +36,9 +36,10 +36,1 +36,8 +37,13 +37,2 +37,6 +37,6 +37,12 +38,5 +38,2 +38,14 +38,5 +38,7 +39,7 +39,15 +39,1 +39,9 +39,15 +40,9 +40,12 +40,5 +40,11 +40,8 +41,8 +41,7 +41,7 +41,4 +41,8 +42,4 +42,8 +42,7 +42,6 +42,6 +43,1 +43,2 +43,14 +43,3 +43,5 +44,4 +44,10 +44,6 +44,7 +44,7 +45,12 +45,3 +45,5 +45,6 +45,15 +46,1 +46,14 +46,3 +46,11 +46,14 +47,13 +47,4 +47,1 +47,13 +47,1 +48,8 +48,3 +48,12 +48,10 +48,13 +49,10 +49,12 +49,11 +49,15 +49,5 +50,13 +50,12 +50,8 +50,11 +50,13 +51,14 +51,10 +51,8 +51,2 +51,12 +52,2 +52,6 +52,14 +52,3 +52,5 +53,4 +53,10 +53,3 +53,14 +53,10 +54,10 +54,1 +54,2 +54,11 +54,13 +55,13 +55,2 +55,7 +55,8 +55,11 +56,7 +56,6 +56,1 +56,15 +56,5 +57,8 +57,4 +57,2 +57,7 +57,1 +58,5 +58,5 +58,11 +58,12 +58,14 +59,13 +59,3 +59,2 +59,14 +59,1 +60,8 +60,14 +60,4 +60,10 +60,2 +61,3 +61,1 +61,2 +61,15 +61,9 +62,13 +62,14 +62,15 +62,10 +62,11 +63,11 +63,5 +63,10 +63,4 +63,7 +64,10 +64,14 +64,14 +64,11 +64,11 +65,15 +65,1 +65,3 +65,6 +65,9 +66,14 +66,3 +66,11 +66,13 +66,14 +67,7 +67,15 +67,14 +67,3 +67,15 +68,10 +68,9 +68,9 +68,6 +68,1 +69,4 +69,6 +69,6 +69,9 +69,4 +70,7 +70,13 +70,6 +70,9 +70,10 +71,14 +71,6 +71,15 +71,4 +71,5 +72,12 +72,1 +72,6 +72,3 +72,13 +73,2 +73,12 +73,15 +73,15 +73,3 +74,4 +74,6 +74,7 +74,2 +74,14 +75,12 +75,4 +75,10 +75,3 +75,12 +76,2 +76,1 +76,15 +76,10 +76,9 +77,6 +77,5 +77,9 +77,14 +77,6 +78,10 +78,2 +78,5 +78,7 +78,9 +79,10 +79,2 +79,9 +79,5 +79,7 +80,12 +80,10 +80,4 +80,14 +80,6 +81,3 +81,3 +81,14 +81,15 +81,6 +82,13 +82,15 +82,14 +82,4 +82,7 +83,14 +83,15 +83,8 +83,14 +83,1 +84,2 +84,13 +84,11 +84,4 +84,11 +85,15 +85,8 +85,15 +85,8 +85,6 +86,12 +86,14 +86,7 +86,5 +86,11 +87,12 +87,4 +87,8 +87,15 +87,14 +88,15 +88,5 +88,1 +88,5 +88,5 +89,14 +89,11 +89,12 +89,3 +89,13 +90,15 +90,7 +90,2 +90,15 +90,9 +91,3 +91,1 +91,15 +91,9 +91,15 +92,12 +92,11 +92,15 +92,15 +92,15 +93,9 +93,9 +93,11 +93,6 +93,15 +94,3 +94,13 +94,2 +94,10 +94,5 +95,9 +95,14 +95,14 +95,4 +95,11 +96,6 +96,1 +96,4 +96,7 +96,10 +97,9 +97,11 +97,14 +97,15 +97,12 +98,10 +98,7 +98,7 +98,12 +98,3 +99,15 +99,1 +99,15 +99,2 +99,15 +100,9 +100,6 +100,11 +100,3 +100,9 +101,8 +101,7 +101,14 +101,1 +101,4 +102,3 +102,8 +102,14 +102,15 +102,13 +103,11 +103,14 +103,7 +103,15 +103,2 +104,13 +104,10 +104,14 +104,8 +104,7 +105,4 +105,15 +105,12 +105,1 +105,2 +106,3 +106,8 +106,9 +106,1 +106,1 +107,11 +107,12 +107,2 +107,13 +107,9 +108,10 +108,10 +108,15 +108,9 +108,15 +109,12 +109,2 +109,14 +109,13 +109,12 +110,4 +110,4 +110,14 +110,5 +110,2 +111,3 +111,3 +111,12 +111,1 +111,4 +112,9 +112,5 +112,9 +112,3 +112,4 +113,6 +113,5 +113,9 +113,7 +113,1 +114,5 +114,9 +114,7 +114,14 +114,8 +115,9 +115,8 +115,6 +115,6 +115,11 +116,8 +116,4 +116,6 +116,2 +116,3 +117,6 +117,2 +117,2 +117,14 +117,4 +118,15 +118,14 +118,11 +118,2 +118,13 +119,14 +119,2 +119,7 +119,5 +119,9 +120,9 +120,4 +120,7 +120,5 +120,4 +121,3 +121,10 +121,6 +121,5 +121,3 +122,4 +122,12 +122,13 +122,7 +122,14 +123,9 +123,15 +123,12 +123,1 +123,1 +124,6 +124,9 +124,11 +124,5 +124,11 +125,12 +125,11 +125,13 +125,14 +125,14 +126,14 +126,15 +126,1 +126,3 +126,10 +127,5 +127,15 +127,8 +127,3 +127,9 +128,15 +128,5 +128,8 +128,15 +128,12 +129,3 +129,14 +129,5 +129,7 +129,9 +130,6 +130,2 +130,4 +130,11 +130,7 +131,6 +131,4 +131,7 +131,5 +131,8 +132,9 +132,3 +132,15 +132,9 +132,8 +133,7 +133,11 +133,8 +133,3 +133,11 +134,11 +134,4 +134,13 +134,8 +134,11 +135,14 +135,7 +135,10 +135,6 +135,4 +136,5 +136,7 +136,7 +136,6 +136,4 +137,1 +137,1 +137,9 +137,13 +137,11 +138,8 +138,14 +138,12 +138,10 +138,7 +139,9 +139,8 +139,8 +139,6 +139,7 +140,14 +140,10 +140,3 +140,3 +140,6 +141,5 +141,14 +141,6 +141,12 +141,3 +142,15 +142,2 +142,10 +142,2 +142,1 +143,12 +143,15 +143,10 +143,3 +143,13 +144,9 +144,6 +144,12 +144,2 +144,5 +145,6 +145,14 +145,7 +145,5 +145,5 +146,15 +146,8 +146,3 +146,12 +146,4 +147,1 +147,10 +147,1 +147,13 +147,2 +148,12 +148,15 +148,8 +148,3 +148,14 +149,14 +149,11 +149,14 +149,9 +149,14 +150,1 +150,10 +150,11 +150,4 +150,11 +151,9 +151,13 +151,15 +151,10 +151,7 +152,2 +152,4 +152,14 +152,4 +152,9 +153,15 +153,13 +153,12 +153,5 +153,15 +154,9 +154,8 +154,8 +154,8 +154,13 +155,10 +155,15 +155,13 +155,5 +155,15 +156,6 +156,10 +156,2 +156,15 +156,2 +157,10 +157,12 +157,5 +157,7 +157,12 +158,13 +158,10 +158,1 +158,10 +158,9 +159,3 +159,6 +159,11 +159,12 +159,11 +160,11 +160,15 +160,11 +160,4 +160,8 +161,4 +161,9 +161,9 +161,4 +161,4 +162,7 +162,14 +162,15 +162,2 +162,14 +163,15 +163,4 +163,12 +163,11 +163,6 +164,5 +164,10 +164,15 +164,12 +164,2 +165,7 +165,10 +165,11 +165,5 +165,11 +166,13 +166,10 +166,9 +166,9 +166,12 +167,15 +167,4 +167,7 +167,8 +167,3 +168,1 +168,15 +168,6 +168,6 +168,12 +169,13 +169,2 +169,1 +169,5 +169,10 +170,9 +170,14 +170,10 +170,12 +170,7 +171,8 +171,5 +171,6 +171,13 +171,13 +172,6 +172,1 +172,1 +172,2 +172,1 +173,4 +173,11 +173,5 +173,4 +173,3 +174,13 +174,11 +174,14 +174,12 +174,13 +175,2 +175,14 +175,11 +175,11 +175,3 +176,5 +176,8 +176,10 +176,9 +176,4 +177,13 +177,2 +177,15 +177,10 +177,15 +178,8 +178,10 +178,5 +178,9 +178,5 +179,11 +179,1 +179,6 +179,15 +179,11 +180,12 +180,10 +180,2 +180,5 +180,14 +181,11 +181,12 +181,14 +181,7 +181,2 +182,8 +182,15 +182,7 +182,6 +182,7 +183,13 +183,7 +183,10 +183,4 +183,12 +184,12 +184,10 +184,14 +184,13 +184,15 +185,8 +185,4 +185,10 +185,5 +185,8 +186,9 +186,3 +186,1 +186,5 +186,9 +187,13 +187,3 +187,15 +187,2 +187,13 +188,15 +188,11 +188,7 +188,5 +188,14 +189,10 +189,14 +189,12 +189,10 +189,15 +190,12 +190,12 +190,8 +190,11 +190,14 +191,11 +191,15 +191,1 +191,15 +191,9 +192,11 +192,8 +192,7 +192,5 +192,15 +193,6 +193,14 +193,7 +193,6 +193,12 +194,6 +194,2 +194,10 +194,6 +194,9 +195,10 +195,13 +195,7 +195,11 +195,1 +196,6 +196,10 +196,1 +196,6 +196,3 +197,7 +197,2 +197,5 +197,4 +197,4 +198,15 +198,15 +198,12 +198,14 +198,14 +199,4 +199,14 +199,8 +199,1 +199,4 +200,2 +200,10 +200,7 +200,15 +200,10 diff --git a/hw0-README.md b/hw0-README.md new file mode 100644 index 0000000..28eab84 --- /dev/null +++ b/hw0-README.md @@ -0,0 +1,295 @@ +# Homework 0: Setup + +This homework is due: **Monday, 1/27/2020, 11:59 PM**. + +## Overview + +This document should help you set up the environment needed to do +assignments in CS 186. + +In order to ensure that your homework solutions are +compatible with the CS186 grading infrastructure, we need to enable you to use +the same software environment that we use for grading. + +To that end, we require that homeworks are implemented to execute correctly inside a [docker +container](https://www.docker.com/resources/what-container), for which we +are providing an image. (Note on terminology: a Docker image is a +static object; a Docker container is a running instance of an image.) You will be able to +run this image on Mac, Windows, or Linux computers. The image itself is an Ubuntu Linux environment +with a bash shell. We assume you know the basics of bash and UNIX commands. + +**Note: we will not offer leniency in cases where your code fails to compile +or provides a different output when our autograder runs it. It is your responsibility +to make sure everything works as expected in the Docker container.** + +## Prerequisites + +No lectures are required to work through this homework. + +## Installing Docker + +The class docker image is called `cs186/environment`. + +Before you can use it, you need to install Docker Community Edition ("CE") on your machine. +Most recent laptops should support Docker CE. + +- To install Docker CE on Mac or Windows, open the +[Docker getting started page](https://www.docker.com/get-started), +stay on the "Developer" tab, and click the button on the right to download the +installer for your OS. Follow all the instructions included. +- To install Docker CE on Linux, open the [Docker +docs](https://docs.docker.com/install/#server), and click the appropriate link +to find instructions for your Linux distro. + +### Additional Notes for Windows Users + +1. Not all editions of Windows support the default `Docker for Windows` distribution. To quote from [the Docker docs](https://docs.docker.com/docker-for-windows/install/): + +> System Requirements: +> +> Windows 10 64bit: Pro, Enterprise or Education (Build 15063 or later). +> Virtualization is enabled in BIOS. Typically, virtualization is enabled by default. This is different from having Hyper-V enabled. For more detail see Virtualization must be enabled in Troubleshooting. +> CPU SLAT-capable feature. +> At least 4GB of RAM. + +2. If you are running another version of Windows, you should be able to get Docker running by downloading [Docker Toolbox](https://docs.docker.com/toolbox/overview/). (Alternatively, if you'd like to upgrade your computer, [UC Berkeley students can install Windows 10 Education for free](https://software.berkeley.edu/microsoft-os)). + +3. When we refer to a "terminal" here, it can be +a traditional `Command Prompt`, or `PowerShell`. However, +interactive docker terminals do not work in `PowerShell ISE`, only +in `PowerShell`. + +4. If you are a Windows +Linux Subsystem user (you would know if you are), then there are +various blog posts [like this one](https://nickjanetakis.com/blog/setting-up-docker-for-windows-and-wsl-to-work-flawlessly) that will +show you how to set WSL to play nicely with Docker for Windows. + +5. You need to run Docker from a Windows account with Administrator privileges. + +## Getting the class docker image + +The next step is to get the class docker image. To do this, +*get on a good internet connection*, open a +terminal on your computer and run + +```bash +docker pull cs186/environment +``` + +This will download the class image onto your computer. When it +completes, try the following to make sure everything is working: + +```bash +docker run cs186/environment echo "hello from cs186" +``` + +That should print `hello from cs186` on your terminal. It did this by +running the `echo` shell command inside a `cs186/environment` docker +container. + +If you are curious to learn more about using Docker, there is [a nice +tutorial online](https://docker-curriculum.com/). + +## Sharing your computer's drive with Docker containers + +An important thing to realize about Docker is that the container has its own +temporary filesystem, which is deleted when the container is terminated, and is +not accessible from outside the container. That means that **you must not store +your files inside the Docker container's filesystem: they will be deleted when +the container is terminated!** + +Instead, you will store your files in your own computer's "local" drive, and *mount* a directory from your local drive within Docker. Mounted volumes in Docker exist outside the Docker container, and hence are not reset when Docker exits. + +You can think of this as: a directory exists on your hard drive, with your CS186 +code. On your computer, perhaps this is located at +`/Users/pusheen/Desktop/cs186` - in other words, on your computer, the path +`/Users/pusheen/Desktop/cs186` points to the directory. Inside your containter, +the `/cs186` path points to the same directory. Since the two paths +(`/Users/pusheen/Desktop/cs186` on your computer and `/cs186` in the container) +point to the exact same directory, any changes you make from either side will be +visible to the other side. + +![diagram of previous paragraph](images/hw0-docker-mounts.png) + +### Configuring support for shared drives + +You may need to configure your Docker installation to share local drives, depending on your OS. Set this up now. + +- **Linux**: you can skip this section -- nothing for you to worry about! +- **Mac**: be aware that you can only share directories under `/Users/`, `/Volumes/`, `/private` and `/tmp/` by default. If that's inconvenient for you, this is [configurable](https://docs.docker.com/docker-for-mac/osxfs/#namespaces). +- **Windows (Docker for Windows)**: To configure Docker to share local drives, follow the instructions [here](https://docs.docker.com/docker-for-windows/#shared-drives). The pathname you will need to use in the `docker run -v` command will need to include a root-level directory name corresponding to the Windows drive letter, and UNIX-style forward slashes. E.g. for the Windows directory `C:\\Users\myid` you would use `docker run -v /c/Users/myid`. +- **Windows (Docker Toolbox)**: be aware that you can only share data within `C:\Users` by default. If the `C:\Users` directory is inconvenient for you, there are [workarounds](http://support.divio.com/local-development/docker/how-to-use-a-directory-outside-cusers-with-docker-toolbox-on-windowsdocker-for-windows) you can try at your own risk. **Also:** the pathname you will need to use in the `docker run -v` command will start with `//c/Users/`. Note the leading double-forward-slash, which is different than Docker for Windows! + +### Mounting your shared drive +Your homework files should live on your machine, *not within a Docker container directory!*. + +First, **on your local machine** (not in Docker), create a cs186 directory +somewhere, and create a file or directory inside (anything -- all that matters +is that the directory is not empty). + +Then, to expose this directory from your machine's filesystem to +your docker container, use the `-v` flag as follows: + +```bash +docker run -v ":/cs186" -it cs186/environment /bin/bash +``` + +For example, if the directory you created was `/Users/pusheen/Desktop/cs186`, +then you would run: + +```bash +docker run -v "/Users/pusheen/Desktop/cs186:/cs186" -it cs186/environment /bin/bash +``` + +(Remember: if you're running Docker Toolbox on Windows, `` should start with `//c/Users/`) + +This mounts your chosen directory to appear inside the Docker container at `/cs186`. + +When you get a prompt from docker, simply `cd` to that directory and you should see your local files. + + ubuntu@95b2c8583144:/$ cd /cs186 + ubuntu@95b2c8583144:/cs186$ ls + + ubuntu@95b2c8583144:/cs186$ exit + +The `ubuntu@95b2c8583144` (hex digits will be different) tells you that you are +currently *inside* the docker container; typing `exit` will leave the docker +container. + +Now you can edit those files within docker *and any changes you make in that directory subtree will +persist across docker invocations* because the files are stored on your machine's filesystem, and not inside the docker container. + +If you do _not_ see anything in the directory (and the directory is not blank on +your machine), double check that you typed the local path in correctly. **You +must use the full path to your directory (starting with `/`, not a relative +path)**. + +Once you have verified that your directory was mounted correctly, run the +following command: + +```bash +docker run --name cs186 -v ":/cs186" -it cs186/environment /bin/bash +``` + +and then exit from the container. This names the container `cs186`, so that you +can start it up without having to fetch maven dependencies every time you work +on a CS 186 homework. If you get an error like `Conflict. The container name +"/cs186" is already in use by container`, run `docker container rm cs186` and +try again. + +You can now start your container whenever you want by running: + +```bash +docker start -ai cs186 +``` + +### Backing up and versioning your work + +We ***very strongly*** encourage you to plan to back up your files using a system +that keeps multiple versions. *You would be crazy not to have a plan for this! We will +not be helping you manage backups, this is your responsibility!* + +The hacker's option here is to [learn `git`](http://git-scm.com/book/en/v1/Getting-Started) well enough to manager your own repository. + +However, since we are not working in teams this semester, it may be sufficient for your +purposes to use an automatic desktop filesync service like Box, OneDrive or Dropbox. +UC Berkeley students have access to free Box.com accounts as documented [here](https://bconnected.berkeley.edu/collaboration-services/box). +There may be some hiccups making sure that your sync software works with Docker shared drives; +for Box users we recommend using the older [Box Sync](https://community.box.com/t5/Using-Box-Sync/Installing-Box-Sync/ta-p/85) instead of the newer Box Drive application. + +**Whatever backup scheme you use, make sure that your files are not publicly visible online. For example, GitHub users should make sure their repos are private.** + +### Using Your Favorite Desktop Tools + +Because your files live on your machine's filesystem (in ``), +you can use your favorite editor or other desktop tools to modify those files. +Any changes you save to those files on your machine will be instantly reflected in Docker. +As a result, you can think of the Docker container as a place to build and run your code, and not a place to *edit* your code! + +**Windows users:** you might need to be aware that Windows convention (inherited from DOS) ends lines of text differently than UNIX/Mac convention +(see [this blog post](https://blog.codinghorror.com/the-great-newline-schism/) for fun history). +This could make your code look odd inside the docker image and not run properly. +If you run into this problem, you may need to configure your editor to generate UNIX-style newlines. + +## `git` and GitHub + +`git` is a *version control* system, that helps developers like you +track different versions of your code, synchronize them across +different machines, and collaborate with others. + +[GitHub](https://github.com) is a site which supports this system, +hosting it as a service. + +We will only be using git and GitHub to pass out homework assignments +in this course. If you don't know much about git, that isn't a +problem: you will *need* to use it only in very simple ways that we will +show you in order to keep up with class assignments. + +Your class docker container includes git already, so you do not need +to install it on your machine separately if you do not want to. +If you'd like to use git for managing your own code versioning, there are many guides to using git online -- +[this](http://git-scm.com/book/en/v1/Getting-Started) is a good one. + +## Fetching the released code + +All assignments in CS 186 will be passed out via GitHub. Please check Piazza to keep up-to-date on changes to assignments. + +You should now follow the instructions in the [Fetching the released code section of +the MOOCbase README](README.md#fetching-the-released-code) to get the skeleton code for +MOOCbase, which most of the homeworks will be building on (HW1 is the sole +exception -- it is on SQL, and will not be using this repo, but this homework and all the remaining homeworks after will +involve writing Java to add functionality to MOOCbase). + +When you exit Docker, you should find the checked-out files on your machine in the +directory `` that you used in your `docker run` command. + +## Getting your environment set up + +This is a good time to go through [Setting up your local development +environment](README.md#setting-up-your-local-development-environment) and +[Running tests](README.md#running-tests). + +To build and test your code in the container, run the following inside the repo +directory: + +```bash +mvn clean test -D HW=0 +``` + +Make sure that everything compiles. There should be 1 failure and 1 test run. + +## Welcome to CS186! + +For this homework, you will need to make a small change to one file. + +Open up `src/main/java/edu/berkeley/cs186/database/databox/StringDataBox.java`. +It's okay if you do not understand most of the code right now. + +The `toString` method currently looks like: +```java + @Override + public String toString() { + return "Welcome to CS186 (original string: " + s + ")"; + } +``` + +Change the method to return only the string (`s`). + +After this, you should now be passing all the tests in `database.databox.TestWelcome`. + +## Submitting the Assignment + +See [the main readme](README.md#submitting-homeworks) for submission instructions. The +homework number for this homework is hw0. + +You should make sure that all code you modify belongs to files with HW0 todo comments in them. +A full list of files that you may modify follows: + +- databox/StringDataBox.java + +## Grading + +- 100% of your grade will be made up of tests released to you (the tests that we + provided in `database.databox.TestWelcome`) and completion of the submission + form on edX. + diff --git a/hw2-README.md b/hw2-README.md new file mode 100644 index 0000000..f0afe16 --- /dev/null +++ b/hw2-README.md @@ -0,0 +1,11 @@ +# Homework 2: B+ Trees + +This homework is due: **Wednesday, 2/26/2020, 11:59 PM**. + +## Overview + +In this homework, you will be implementing B+ tree indices. + +## Getting Started + +This homework will be released on **Sunday, 2/9/2020**. diff --git a/hw3-README.md b/hw3-README.md new file mode 100644 index 0000000..9a56f99 --- /dev/null +++ b/hw3-README.md @@ -0,0 +1,31 @@ +# Homework 3: Joins and Query Optimization + +This homework is divided into two parts. + +Part 1 is due: **Sunday, 3/8/2020, 11:59 PM**. + +Part 2 is due: **Monday, 3/16/2020, 11:59 PM**. + +Part 2 does not require Part 1 to be completed first, but you will need to make sure +any code you write for Part 1 does not throw an exception before starting Part 2. See +the Grading section at the bottom of this document for notes on how your score will +be computed. + +**You will not be able to use slip minutes for the Part 1 deadline.** The standard late penalty +(33%/day, counted in days not minutes) will apply after the Part 1 deadline. Slip minutes +may be used for the Part 2 deadline, and the late penalty for the two parts are independent. + +For example, if you submit Part 1 at 5:30 AM two days after it is due, +and Part 2 at 6:00 PM the day after it is due, you will recieve: +- 66% penalty on your Part 1 submission +- No penalty on your Part 2 submission +- A total of 1080 slip minutes consumed + +## Overview + +In this assignment, you will implement some join algorithms and a limited +version of the Selinger optimizer. + +## Getting Started + +This homework will be released on **Friday, 2/28/2020**. diff --git a/hw4-README.md b/hw4-README.md new file mode 100644 index 0000000..c2de171 --- /dev/null +++ b/hw4-README.md @@ -0,0 +1,29 @@ +# Homework 4: Locking + +This homework is divided into two parts. + +Part 1 is due: **Friday, 4/3/2020, 11:59 PM**. + +Part 2 is due: **Tuesday, 4/14/2020, 11:59 PM**. + +Part 2 requires Part 1 to be completed first. See the Grading section at the +bottom of this document for notes on how your score will be computed. + +**You will not be able to use slip minutes for the Part 1 deadline.** The standard late penalty +(33%/day, counted in days not minutes) will apply after the Part 1 deadline. Slip minutes +may be used for the Part 2 deadline, and the late penalty for the two parts are independent. + +For example, if you submit Part 1 at 5:30 AM two days after it is due, +and Part 2 at 6:00 PM the day after it is due, you will recieve: +- 66% penalty on your Part 1 submission +- No penalty on your Part 2 submission +- A total of 1080 slip minutes consumed + +## Overview + +In this assignment, you will implement multigranularity locking and integrate +it into the codebase. + +## Getting Started + +This homework will be released on **Wednesday, 3/18/2020**. diff --git a/hw5-README.md b/hw5-README.md new file mode 100644 index 0000000..1d1b19d --- /dev/null +++ b/hw5-README.md @@ -0,0 +1,11 @@ +# Homework 5: Recovery + +This homework is due: **Friday, 5/1/2020, 11:59 PM**. + +## Overview + +In this assignment, you will implement write-ahead logging. + +## Getting Started + +This homework will be released on **Thursday, 4/16/2020**. diff --git a/images/hw0-docker-mounts.png b/images/hw0-docker-mounts.png new file mode 100644 index 0000000..b47d13e Binary files /dev/null and b/images/hw0-docker-mounts.png differ diff --git a/images/intellij-empty-configuration.png b/images/intellij-empty-configuration.png new file mode 100644 index 0000000..3c3e9b8 Binary files /dev/null and b/images/intellij-empty-configuration.png differ diff --git a/images/intellij-filledin-configuration.png b/images/intellij-filledin-configuration.png new file mode 100644 index 0000000..704ba3c Binary files /dev/null and b/images/intellij-filledin-configuration.png differ diff --git a/intellij-test-setup.md b/intellij-test-setup.md new file mode 100644 index 0000000..8210ae3 --- /dev/null +++ b/intellij-test-setup.md @@ -0,0 +1,16 @@ +# IntelliJ setup for running tests + +This document will help you set up IntelliJ for running a homework's tests +(making it run all the tests that `mvn test -DHW=X` would run). + +1. Open up Run/Debug Configurations with Run > Edit Configurations. +2. Click the + button in the top left to create a new configuration, and choose JUnit from + the dropdown. This should get you the following unnamed configuration: + ![unnamed configuration menu](images/intellij-empty-configuration.png) +3. Fill in the fields as listed below, then press OK. + ![filled in menu](images/intellij-filledin-configuration.png) + - Name: HW2 tests (or whichever homework you're setting up) + - Test kind: Category + - Category: edu.berkeley.cs186.database.categories.HW2Tests (or the category corresponding to the homework you're setting up) + - Search for tests: In whole project +4. You should now see HW2 tests in the dropdown in the top right. You can run/debug this configuration to run all the HW2 tests. diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..6803eba --- /dev/null +++ b/pom.xml @@ -0,0 +1,183 @@ + + + 4.0.0 + edu.berkeley.cs186 + database + 1.0-SNAPSHOT + + + + junit + junit + 4.12 + test + + + + + 1.8 + 1.8 + UTF-8 + UTF-8 + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.surefire + surefire-junit47 + 2.20.1 + + + 2.20.1 + + + **/TestUtils.java + **/TestSourceOperator.java + **/*$* + + false + -Xms32m -Xmx32m + edu.berkeley.cs186.database.categories.HW${HW}Tests + + + + + + + + public + + + + org.apache.maven.plugins + maven-surefire-plugin + + edu.berkeley.cs186.database.categories.HiddenTests,edu.berkeley.cs186.database.categories.SystemTests,edu.berkeley.cs186.database.categories.StudentTests,edu.berkeley.cs186.database.categories.StudentTestRunner + + + + + + + hidden + + + + org.apache.maven.plugins + maven-surefire-plugin + + edu.berkeley.cs186.database.categories.PublicTests,edu.berkeley.cs186.database.categories.SystemTests,edu.berkeley.cs186.database.categories.StudentTests,edu.berkeley.cs186.database.categories.StudentTestRunner + + + + + + + student + + + + org.apache.maven.plugins + maven-surefire-plugin + + edu.berkeley.cs186.database.categories.PublicTests,edu.berkeley.cs186.database.categories.HiddenTests,edu.berkeley.cs186.database.categories.SystemTests,edu.berkeley.cs186.database.categories.StudentTestRunner + + + + + + + student-runner + + + + org.apache.maven.plugins + maven-surefire-plugin + + edu.berkeley.cs186.database.categories.PublicTests,edu.berkeley.cs186.database.categories.HiddenTests,edu.berkeley.cs186.database.categories.SystemTests,edu.berkeley.cs186.database.categories.StudentTests + + + + + + + system + + + + org.apache.maven.plugins + maven-surefire-plugin + + edu.berkeley.cs186.database.categories.PublicTests,edu.berkeley.cs186.database.categories.HiddenTests,edu.berkeley.cs186.database.categories.StudentTests,edu.berkeley.cs186.database.categories.StudentTestRunner + + + + + + + nonsystem + + + + org.apache.maven.plugins + maven-surefire-plugin + + edu.berkeley.cs186.database.categories.SystemTests + + + + + + true + + + + uncategorized + + + + org.apache.maven.plugins + maven-surefire-plugin + + edu.berkeley.cs186.database.categories.SystemTests,edu.berkeley.cs186.database.categories.PublicTests,edu.berkeley.cs186.database.categories.HiddenTests,edu.berkeley.cs186.database.categories.StudentTests,edu.berkeley.cs186.database.categories.StudentTestRunner + + + + + + + all + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 2.8 + + + + diff --git a/public.key b/public.key new file mode 100644 index 0000000..ecb9621 --- /dev/null +++ b/public.key @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF4mo5kBEAC+ZHsiOQ0cfW96+eN18sxYzEIAFqfCXzrBl45IWK2Z2ObPFSLZ +OcccQ7md1KcXlWTWgprZy4zExwT8+1+2dpc4pGlOcda/vuTQjgxYW0MMUbaMv817 +3Jrq6MaKLq9FxLI9VCz+f9un8KW8WkW8lxDmL+wXJY/ziWh7EN2ubNzutX1xLei6 +ou28uxgMsgNE5wRhk768gycY77Lhy1y3nxNZtQlINfCqE05rFyuAfqIvXTNHRTVu +gH+gn+Xtk/H4FSg08JJkq2SAj50DqoB9CgwouC8aB1BRL7rZlrw3287R0I4AVDvH +9217jgOo5SbKmHbbf0zjW3azSX0/bYZSDMVXNLLi1hXKXFCWMlqoejllDHTkjNHY +4G5PDKh8lLspif46ir0SBsO1IP1cDYUScL/h1qyygaot5DEFsJ+91qsH8Yqw0K+y ++gI9E9xFEqMurijawFPuGVqeb3tgoxEbkp9UwMIEkddlL6hg/SrDeFTgqEewkgNd +jCquOdLF1zANK14LLMJ1GUs/6RvB+GUnqD/humJ2O+/2aYrkOVRKGU0+VhU7jalc +2vgOx4lQbfQG284sayDdOiqF3f2rFTzcuPECJNWdoVD5WCJMWbs1YiS9nfH5qeeM +d0Hrul3/aCjgrkn4VFHmU2Pw6I9UwIOWoPGb8qepHonpyU3mfuoyL8n+3wARAQAB +tD9DUzE4NiBTdGFmZiAoQ1MxODYgU3ByaW5nIDIwMjAgR1BHIGtleXMpIDxka2lt +OTA4QGJlcmtlbGV5LmVkdT6JAlQEEwEIAD4WIQSkmJK9nX1On7ooNjZ7Q6fP+piP +iAUCXiajmQIbAwUJAO1OAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRB7Q6fP ++piPiJHrD/9Gvet1SVfYkCl8t7r96smKguI6ZLdMCylmv6UumgfJnIJ4xIOR8Cff +U4dWjexbRhTlbntBquQGsu9UiC8oM5ExTqbZ1Z6Hri4CYoys++1X7RDY42pEic95 +XnnNA2rZUhqaTjalnvCdEtQFQr6RD3z9Jhm0HSvta1AlxXtmu0HuoPd4MVLbj6Zq +//mI7VVV1TP5jvIG7HCG4nsPYhpxwLqv6eoqiSsoslx1OES12BSr9e6JX+a8Vfkd +EJbu7axYoh7euuLNPcRkMdxElEUr7jtPiK256dr0/XZJ1AAN8h8KpB/NDPTrmcyM +L60skvb0UhnGuhVVzbLSR4lMHpH9W82prcMFf/hlLQAQQpzjcCSrWRFwyPPXMOtE +4sXhC3nBSO782JejzlSjRrvYpn6dVADEhj6ul9aGCSwki+YiSUg4Fa2T91qk0Qbu +mEZrNyePn9hzQAy7ZY4yZqM97GduQ1TesOnfqilMsiUUg7R18b3yNEemErAdrHj+ ++SbxceQD9b+qb9hRjs/Qx+xPWqwaOcWvYNNVJJtDhW0N+s31L6czbxv+6COETRW4 +DhmpwphNZuWwbLXp2nJWPebcjt4wfTIwfq75f1Sa373WoOiBnNA75AXjBUPVioWS +WXXwyghYH3NFBhzTV2nMKgjUlaYqtMpPcKcqWfFM9iuFFkqfrQAz87kCDQReJqOZ +ARAAyyJpxfL4clNhsUnhLhdJRaNQdeVDW5FtQYtq1vtLf81yvIbY7vQmGyDc1sdO +63c4oiNRl66h+zRnrLSYfsZN4HRfaIZ/cPROV60JvnGxHj3cSopQnhJdpJ/A5OUF +eQEcv6YhT7iwjWB254wIE4KhE4+S2aRd3J526WWY8nIyCtcpLN73liKK2RU824Fu +gNiElXOhxhqMWkBkCR7zkG3ZbZPaB6cFihCqZznjRMoQu8iEst74isxMfLlmHAZH +F4B9ji0jGJtdbabuxDSIx7lYgBqSB5zrDiHaiawCbFRAFhi08cbvRs6qeWfQ3jBY +KR3P78XBnCBdNbACbnB0hrfGWmKSSpQDMPvnf+dXcZAQQTH+SazssjXnqsIUCl0O +LHlUY9EbCGIz5TUtSIL/x/1f4mdfiv7seuaiLIKe5L4/3Q+GSmI9eWwrBgVfngxY +eh9ZRMQJwGYPXnq5qc9yponFTVpY7L3/EH+gahZSPYm7TOsPaH4jmjITmiWiPYSn +kGFarnb7bQaA7IW5VYw5qIY6+JQgqz6X+yfRn61oMM15vYkaPhBKk1Na7rR2K9Kd +jhMwZrnSKjbdZoBIbQGyZajWqPIXbgRtnDIWQ2xbj/nQME9dc4+vChMkp3lNEw+1 +bpdhShlPyKa8hSKXudbhQvL/xPEBNGV+q9dE7CYMnSnLvQUAEQEAAYkCPAQYAQgA +JhYhBKSYkr2dfU6fuig2NntDp8/6mI+IBQJeJqOZAhsMBQkA7U4AAAoJEHtDp8/6 +mI+IJzQP/i5HTweMUai58cg2yoPcS6Pk1UUNLjKGuWS2eliwhHfbGm2Ezv56xF3S +0CQVynpILs2DZl5h/6RvOPHtX9wh9OHkWHrnm8Te9faA7yW06laun6oM1aBvn/yS +v71h0FYMpofUCL6+TuQRUbovtqZa8rDWJeOXsD3ee1EiiehAzX4jWpSFHNTJTT9k +uHtUea5S77nOv6cVMHhYqJ5aq5FC360ZQjKxtH+gNajake7u43XHOm+cnIvcAO3Z +lfkntC7pwuAUXz8uITuRYOSof+cxdV/zmUvc02FY8PhSgbedbEINhystQpzCGsOh +LZzk0mi3jgKyYcayEl07tB1X1xWXKtvredHeE3dhHy4IddSqoAM2Cj5zHC09fwXr +uP6XtsenoihZuk2/1AmHO4VrfOXDCDwWwyUN9AUXmc+cU4O4z6rrytxqL7zPdwK3 +wKxXeFVIHMMy8PopcCdb3fU3vWJTW7fkfv7LbEIEf2EjRqVIZo1Fet00yb9v1ci3 +WFA7z0DlIcBTIrZcE1iMKKQnWwKhoPrbU+38M+Pq2xiguBmk/te0jIRbIlUX4vYX ++p6UV2b4J3/miHi4cjhGVJBnJSuBoWalU0VHq/zfBRKga+10VV0vgzI9flN4DLcP +lyAZIs62VaV/4IPgF/BMC6Poxokme8e4VSGlfeO4O89r1e2xs+wY +=GCpj +-----END PGP PUBLIC KEY BLOCK----- diff --git a/src/main/java/edu/berkeley/cs186/database/AbstractTransaction.java b/src/main/java/edu/berkeley/cs186/database/AbstractTransaction.java new file mode 100644 index 0000000..a5f80ce --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/AbstractTransaction.java @@ -0,0 +1,65 @@ +package edu.berkeley.cs186.database; + +public abstract class AbstractTransaction implements Transaction { + private Status status = Status.RUNNING; + + /** + * Called when commit() is called. Any exception thrown in this method will cause + * the transaction to abort. + */ + protected abstract void startCommit(); + + /** + * Called when rollback() is called. No exception should be thrown, and any exception + * thrown will be interpreted the same as if the method had returned normally. + */ + protected abstract void startRollback(); + + /** + * Commit the transaction. + */ + @Override + public final void commit() { + if (status != Status.RUNNING) { + throw new IllegalStateException("transaction not in running state, cannot commit"); + } + startCommit(); + } + + /** + * Rollback the transaction. + */ + @Override + public final void rollback() { + if (status != Status.RUNNING) { + throw new IllegalStateException("transaction not in running state, cannot rollback"); + } + startRollback(); + } + + @Override + public final Status getStatus() { + return status; + } + + @Override + public void setStatus(Status status) { + this.status = status; + } + + /** + * Implements close() as commit() when abort/commit not called - so that we can write: + * + * try (Transaction t = ...) { + * ... + * } + * + * and have the transaction commit. + */ + @Override + public final void close() { + if (status == Status.RUNNING) { + commit(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/AbstractTransactionContext.java b/src/main/java/edu/berkeley/cs186/database/AbstractTransactionContext.java new file mode 100644 index 0000000..fffb17e --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/AbstractTransactionContext.java @@ -0,0 +1,76 @@ +package edu.berkeley.cs186.database; + +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** + * This transaction context implementation assumes that exactly one transaction runs + * on a thread at a time, and that, aside from the unblock() method, no methods + * of the transaction are called from a different thread than the thread that the + * transaction is associated with. This implementation blocks the thread when + * block() is called. + */ +public abstract class AbstractTransactionContext implements TransactionContext { + private boolean blocked = false; + private boolean startBlock = false; + private final ReentrantLock transactionLock = new ReentrantLock(); + private final Condition unblocked = transactionLock.newCondition(); + + /** + * prepareBlock acquires the lock backing the condition variable that the transaction + * waits on. Must be called before block(), and is used to ensure that the unblock() call + * corresponding to the following block() call cannot be run before the transaction blocks. + */ + @Override + public void prepareBlock() { + if (this.startBlock) { + throw new IllegalStateException("already preparing to block"); + } + this.transactionLock.lock(); + this.startBlock = true; + } + + /** + * Blocks the transaction (and thread). prepareBlock() must be called first. + */ + @Override + public void block() { + if (!this.startBlock) { + throw new IllegalStateException("prepareBlock() must be called before block()"); + } + try { + this.blocked = true; + while (this.blocked) { + this.unblocked.awaitUninterruptibly(); + } + } finally { + this.startBlock = false; + this.transactionLock.unlock(); + } + } + + /** + * Unblocks the transaction (and thread running the transaction). + */ + @Override + public void unblock() { + this.transactionLock.lock(); + try { + this.blocked = false; + this.unblocked.signal(); + } finally { + this.transactionLock.unlock(); + } + } + + @Override + public boolean getBlocked() { + return this.blocked; + } + + @SuppressWarnings("unchecked") + private static void rethrow(Throwable t) throws T { + // rethrows checked exceptions as unchecked + throw (T) t; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/Database.java b/src/main/java/edu/berkeley/cs186/database/Database.java new file mode 100644 index 0000000..6be0b71 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/Database.java @@ -0,0 +1,1490 @@ +package edu.berkeley.cs186.database; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.*; +import java.util.function.UnaryOperator; + +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.*; +import edu.berkeley.cs186.database.databox.*; +import edu.berkeley.cs186.database.index.BPlusTree; +import edu.berkeley.cs186.database.index.BPlusTreeMetadata; +import edu.berkeley.cs186.database.io.*; +import edu.berkeley.cs186.database.memory.*; +import edu.berkeley.cs186.database.query.QueryPlan; +import edu.berkeley.cs186.database.query.QueryPlanException; +import edu.berkeley.cs186.database.query.SortOperator; +import edu.berkeley.cs186.database.recovery.*; +import edu.berkeley.cs186.database.table.*; +import edu.berkeley.cs186.database.table.stats.TableStats; + +@SuppressWarnings("unused") +public class Database implements AutoCloseable { + private static final String METADATA_TABLE_PREFIX = "information_schema."; + private static final String TABLE_INFO_TABLE_NAME = METADATA_TABLE_PREFIX + "tables"; + private static final String INDEX_INFO_TABLE_NAME = METADATA_TABLE_PREFIX + "indices"; + private static final int DEFAULT_BUFFER_SIZE = 262144; // default of 1G + private static final int MAX_SCHEMA_SIZE = 4005; // a wonderful number pulled out of nowhere + + // information_schema.tables, manages all tables in the database + private Table tableInfo; + // information_schema.indices, manages all indices in the database + private Table indexInfo; + // table name to table object mapping + private final ConcurrentMap tableLookup; + // index name to bplustree object mapping (index name is: "table,col") + private final ConcurrentMap indexLookup; + // table name to record id of entry in tableInfo + private final ConcurrentMap tableInfoLookup; + // index name to record id of entry in indexInfo + private final ConcurrentMap indexInfoLookup; + // list of indices for each table + private final ConcurrentMap> tableIndices; + + // number of transactions created + private long numTransactions; + + // lock manager + private final LockManager lockManager; + // disk space manager + private final DiskSpaceManager diskSpaceManager; + // buffer manager + private final BufferManager bufferManager; + // recovery manager + private final RecoveryManager recoveryManager; + + // transaction for creating metadata partitions and loading tables + private final Transaction primaryInitTransaction; + // transaction for loading indices + private final Transaction secondaryInitTransaction; + // thread pool for background tasks + private final ExecutorService executor; + + // number of pages of memory to use for joins, etc. + private int workMem = 1024; // default of 4M + // number of pages of memory available total + private int numMemoryPages; + + // progress in loading tables/indices + private final Phaser loadingProgress = new Phaser(1); + // active transactions + private Phaser activeTransactions = new Phaser(0); + + /** + * Creates a new database with locking disabled. + * + * @param fileDir the directory to put the table files in + */ + public Database(String fileDir) { + this (fileDir, DEFAULT_BUFFER_SIZE); + } + + /** + * Creates a new database with locking disabled. + * + * @param fileDir the directory to put the table files in + * @param numMemoryPages the number of pages of memory in the buffer cache + */ + public Database(String fileDir, int numMemoryPages) { + this(fileDir, numMemoryPages, new DummyLockManager()); + waitSetupFinished(); + } + + /** + * Creates a new database. + * + * @param fileDir the directory to put the table files in + * @param numMemoryPages the number of pages of memory in the buffer cache + * @param lockManager the lock manager + */ + public Database(String fileDir, int numMemoryPages, LockManager lockManager) { + this(fileDir, numMemoryPages, lockManager, new ClockEvictionPolicy()); + } + + /** + * Creates a new database. + * + * @param fileDir the directory to put the table files in + * @param numMemoryPages the number of pages of memory in the buffer cache + * @param lockManager the lock manager + * @param policy eviction policy for buffer cache + */ + public Database(String fileDir, int numMemoryPages, LockManager lockManager, + EvictionPolicy policy) { + boolean initialized = setupDirectory(fileDir); + + numTransactions = 0; + this.numMemoryPages = numMemoryPages; + this.lockManager = lockManager; + tableLookup = new ConcurrentHashMap<>(); + indexLookup = new ConcurrentHashMap<>(); + tableIndices = new ConcurrentHashMap<>(); + tableInfoLookup = new ConcurrentHashMap<>(); + indexInfoLookup = new ConcurrentHashMap<>(); + this.executor = new ThreadPool(); + + // TODO(hw5): change to use ARIES recovery manager + recoveryManager = new DummyRecoveryManager(); + //recoveryManager = new ARIESRecoveryManager(lockManager.databaseContext(), + // this::beginRecoveryTranscation, this::setTransactionCounter, this::getTransactionCounter); + + diskSpaceManager = new DiskSpaceManagerImpl(fileDir, recoveryManager); + bufferManager = new BufferManagerImpl(diskSpaceManager, recoveryManager, numMemoryPages, + policy); + + if (!initialized) { + diskSpaceManager.allocPart(0); + } + + recoveryManager.setManagers(diskSpaceManager, bufferManager); + + if (!initialized) { + recoveryManager.initialize(); + } + + Runnable r = recoveryManager.restart(); + executor.submit(r); + + primaryInitTransaction = beginTransaction(); + secondaryInitTransaction = beginTransaction(); + TransactionContext.setTransaction(primaryInitTransaction.getTransactionContext()); + + if (!initialized) { + // create log partition, information_schema.tables partition, and information_schema.indices partition + diskSpaceManager.allocPart(1); + diskSpaceManager.allocPart(2); + } + + TransactionContext.unsetTransaction(); + LockContext dbContext = lockManager.databaseContext(); + LockContext tableInfoContext = getTableInfoContext(); + + if (!initialized) { + dbContext.acquire(primaryInitTransaction.getTransactionContext(), LockType.X); + this.initTableInfo(); + this.initIndexInfo(); + this.loadingProgress.arriveAndDeregister(); + } else { + this.loadMetadataTables(); + this.loadTablesAndIndices(); + } + } + + private boolean setupDirectory(String fileDir) { + File dir = new File(fileDir); + boolean initialized = dir.exists(); + if (!initialized) { + if (!dir.mkdir()) { + throw new DatabaseException("failed to create directory " + fileDir); + } + } else if (!dir.isDirectory()) { + throw new DatabaseException(fileDir + " is not a directory"); + } + try (DirectoryStream dirStream = Files.newDirectoryStream(dir.toPath())) { + initialized = initialized && dirStream.iterator().hasNext(); + } catch (IOException e) { + throw new DatabaseException(e); + } + return initialized; + } + + // create information_schema.tables + private void initTableInfo() { + TransactionContext.setTransaction(primaryInitTransaction.getTransactionContext()); + + long tableInfoPage0 = DiskSpaceManager.getVirtualPageNum(1, 0); + diskSpaceManager.allocPage(tableInfoPage0); + + LockContext tableInfoContext = getTableInfoContext(); + HeapFile tableInfoHeapFile = new PageDirectory(bufferManager, 1, tableInfoPage0, (short) 0, + tableInfoContext); + tableInfo = new Table(TABLE_INFO_TABLE_NAME, getTableInfoSchema(), tableInfoHeapFile, + tableInfoContext); + tableInfo.disableAutoEscalate(); + tableInfoLookup.put(TABLE_INFO_TABLE_NAME, tableInfo.addRecord(Arrays.asList( + new StringDataBox(TABLE_INFO_TABLE_NAME, 32), + new IntDataBox(1), + new LongDataBox(tableInfoPage0), + new BoolDataBox(false), + new StringDataBox(new String(getTableInfoSchema().toBytes()), MAX_SCHEMA_SIZE) + ))); + tableLookup.put(TABLE_INFO_TABLE_NAME, tableInfo); + tableIndices.put(TABLE_INFO_TABLE_NAME, Collections.emptyList()); + + primaryInitTransaction.commit(); + TransactionContext.unsetTransaction(); + } + + // create information_schema.indices + private void initIndexInfo() { + TransactionContext.setTransaction(secondaryInitTransaction.getTransactionContext()); + + long indexInfoPage0 = DiskSpaceManager.getVirtualPageNum(2, 0); + diskSpaceManager.allocPage(indexInfoPage0); + + LockContext indexInfoContext = getIndexInfoContext(); + HeapFile heapFile = new PageDirectory(bufferManager, 2, indexInfoPage0, (short) 0, + indexInfoContext); + indexInfo = new Table(INDEX_INFO_TABLE_NAME, getIndexInfoSchema(), heapFile, indexInfoContext); + indexInfo.disableAutoEscalate(); + indexInfo.setFullPageRecords(); + tableInfoLookup.put(INDEX_INFO_TABLE_NAME, tableInfo.addRecord(Arrays.asList( + new StringDataBox(INDEX_INFO_TABLE_NAME, 32), + new IntDataBox(2), + new LongDataBox(indexInfoPage0), + new BoolDataBox(false), + new StringDataBox(new String(getIndexInfoSchema().toBytes()), MAX_SCHEMA_SIZE) + ))); + tableLookup.put(INDEX_INFO_TABLE_NAME, indexInfo); + tableIndices.put(INDEX_INFO_TABLE_NAME, Collections.emptyList()); + + secondaryInitTransaction.commit(); + TransactionContext.unsetTransaction(); + } + + private void loadMetadataTables() { + // load information_schema.tables + LockContext tableInfoContext = getTableInfoContext(); + HeapFile tableInfoHeapFile = new PageDirectory(bufferManager, 1, + DiskSpaceManager.getVirtualPageNum(1, 0), (short) 0, tableInfoContext); + tableInfo = new Table(TABLE_INFO_TABLE_NAME, getTableInfoSchema(), tableInfoHeapFile, + tableInfoContext); + tableInfo.disableAutoEscalate(); + tableLookup.put(TABLE_INFO_TABLE_NAME, tableInfo); + tableIndices.put(TABLE_INFO_TABLE_NAME, Collections.emptyList()); + // load information_schema.indices + LockContext indexInfoContext = getIndexInfoContext(); + HeapFile indexInfoHeapFile = new PageDirectory(bufferManager, 2, + DiskSpaceManager.getVirtualPageNum(2, 0), (short) 0, indexInfoContext); + indexInfo = new Table(INDEX_INFO_TABLE_NAME, getIndexInfoSchema(), indexInfoHeapFile, + indexInfoContext); + indexInfo.disableAutoEscalate(); + indexInfo.setFullPageRecords(); + tableLookup.put(INDEX_INFO_TABLE_NAME, indexInfo); + tableIndices.put(INDEX_INFO_TABLE_NAME, Collections.emptyList()); + } + + // load tables from information_schema.tables + private void loadTablesAndIndices() { + Iterator iter = tableInfo.ridIterator(); + + LockContext dbContext = lockManager.databaseContext(); + LockContext tableInfoContext = getTableInfoContext(); + TransactionContext primaryTC = primaryInitTransaction.getTransactionContext(); + + dbContext.acquire(primaryTC, LockType.IX); + tableInfoContext.acquire(primaryTC, LockType.IX); + + for (RecordId recordId : (Iterable) () -> iter) { + TransactionContext.setTransaction(primaryTC); + + try { + LockContext tableMetadataContext = tableInfoContext.childContext(recordId.getPageNum()); + // need an X lock here even though we're only reading, to prevent others from attempting to + // fetch table object before it has been constructed + tableMetadataContext.acquire(primaryTC, LockType.X); + + TableInfoRecord record = new TableInfoRecord(tableInfo.getRecord(recordId)); + if (!record.isAllocated()) { + tableInfo.deleteRecord(recordId); + continue; + } + + if (record.isTemporary) { + continue; // no need to load temp tables - they will be cleaned up eventually by recovery + } + + tableInfoLookup.put(record.tableName, recordId); + tableIndices.putIfAbsent(record.tableName, Collections.synchronizedList(new ArrayList<>())); + + if (record.tableName.startsWith(METADATA_TABLE_PREFIX)) { + tableMetadataContext.release(primaryTC); + continue; + } + + loadingProgress.register(); + executor.execute(() -> { + loadingProgress.arriveAndAwaitAdvance(); + TransactionContext.setTransaction(primaryInitTransaction.getTransactionContext()); + + // X(table) acquired during table ctor; not needed earlier because no one can even check + // if table exists due to X(table metadata) lock + LockContext tableContext = getTableContext(record.tableName, record.partNum); + HeapFile heapFile = new PageDirectory(bufferManager, record.partNum, record.pageNum, (short) 0, + tableContext); + Table table = new Table(record.tableName, record.schema, heapFile, tableContext); + tableLookup.put(record.tableName, table); + + // sync on lock manager to ensure that multiple jobs don't + // try to perform LockContext operations for the same transaction simultaneously + synchronized (lockManager) { + LockContext metadataContext = getTableInfoContext().childContext(recordId.getPageNum()); + + tableContext.release(primaryTC); + metadataContext.release(primaryTC); + } + + TransactionContext.unsetTransaction(); + loadingProgress.arriveAndDeregister(); + }); + } finally { + TransactionContext.unsetTransaction(); + } + } + + this.loadIndices(); + + loadingProgress.arriveAndAwaitAdvance(); // start table/index loading + + executor.execute(() -> { + loadingProgress.arriveAndAwaitAdvance(); // wait for all tables and indices to load + primaryInitTransaction.commit(); + secondaryInitTransaction.commit(); + loadingProgress.arriveAndDeregister(); + // add toggleable auto-escalate + }); + } + + // load indices from information_schema.indices + private void loadIndices() { + Iterator iter = indexInfo.ridIterator(); + + LockContext dbContext = lockManager.databaseContext(); + LockContext tableInfoContext = getTableInfoContext(); + LockContext indexInfoContext = getIndexInfoContext(); + TransactionContext secondaryTC = secondaryInitTransaction.getTransactionContext(); + + dbContext.acquire(secondaryTC, LockType.IX); + tableInfoContext.acquire(secondaryTC, LockType.IS); + indexInfoContext.acquire(secondaryTC, LockType.IX); + + for (RecordId recordId : (Iterable) () -> iter) { + LockContext indexMetadataContext = indexInfoContext.childContext(recordId.getPageNum()); + // need an X lock here even though we're only reading, to prevent others from attempting to + // fetch index object before it has been constructed + indexMetadataContext.acquire(secondaryTC, LockType.X); + + BPlusTreeMetadata metadata = parseIndexMetadata(indexInfo.getRecord(recordId)); + if (metadata == null) { + indexInfo.deleteRecord(recordId); + return; + } + + loadingProgress.register(); + executor.execute(() -> { + RecordId tableMetadataRid = tableInfoLookup.get(metadata.getTableName()); + LockContext tableMetadataContext = tableInfoContext.childContext(tableMetadataRid.getPageNum()); + tableMetadataContext.acquire(secondaryTC, LockType.S); // S(metadata) + LockContext tableContext = getTableContext(metadata.getTableName()); + tableContext.acquire(secondaryTC, LockType.S); // S(table) + + loadingProgress.arriveAndAwaitAdvance(); + + String indexName = metadata.getName(); + LockContext indexContext = getIndexContext(indexName, metadata.getPartNum()); + + try { + BPlusTree tree = new BPlusTree(bufferManager, metadata, indexContext); + if (!tableIndices.containsKey(metadata.getTableName())) { + // the list only needs to be synchronized while indices are being loaded, as multiple + // indices may attempt to add themselves to the list at the same time + tableIndices.put(metadata.getTableName(), Collections.synchronizedList(new ArrayList<>())); + } + tableIndices.get(metadata.getTableName()).add(indexName); + indexLookup.put(indexName, tree); + indexInfoLookup.put(indexName, recordId); + + synchronized (loadingProgress) { + indexContext.release(secondaryInitTransaction.getTransactionContext()); + } + } finally { + loadingProgress.arriveAndDeregister(); + } + }); + } + loadingProgress.arriveAndAwaitAdvance(); // start index loading + } + + // wait until setup has finished + public void waitSetupFinished() { + while (!loadingProgress.isTerminated()) { + loadingProgress.awaitAdvance(loadingProgress.getPhase()); + } + } + + // wait for all transactions to finish + public synchronized void waitAllTransactions() { + while (!activeTransactions.isTerminated()) { + activeTransactions.awaitAdvance(activeTransactions.getPhase()); + } + } + + /** + * Close this database. + */ + @Override + public synchronized void close() { + if (this.executor.isShutdown()) { + return; + } + + // wait for all transactions to terminate + this.waitAllTransactions(); + + // finish executor tasks + this.executor.shutdown(); + + this.bufferManager.evictAll(); + + this.recoveryManager.close(); + + this.tableInfo = null; + this.indexInfo = null; + + this.tableLookup.clear(); + this.indexLookup.clear(); + this.tableInfoLookup.clear(); + this.indexInfoLookup.clear(); + this.tableIndices.clear(); + + this.bufferManager.close(); + this.diskSpaceManager.close(); + } + + public ExecutorService getExecutor() { + return executor; + } + + public LockManager getLockManager() { + return lockManager; + } + + public DiskSpaceManager getDiskSpaceManager() { + return diskSpaceManager; + } + + public BufferManager getBufferManager() { + return bufferManager; + } + + @Deprecated + public Table getTable(String tableName) { + return tableLookup.get(prefixUserTableName(tableName)); + } + + public int getWorkMem() { + // cap work memory at number of memory pages -- this is likely to cause out of memory + // errors if actually set this high + return this.workMem > this.numMemoryPages ? this.numMemoryPages : this.workMem; + } + + public void setWorkMem(int workMem) { + this.workMem = workMem; + } + + // schema for information_schema.tables + private Schema getTableInfoSchema() { + return new Schema( + Arrays.asList("table_name", "part_num", "page_num", "is_temporary", "schema"), + Arrays.asList(Type.stringType(32), Type.intType(), Type.longType(), Type.boolType(), + Type.stringType(MAX_SCHEMA_SIZE)) + ); + } + + // schema for information_schema.indices + private Schema getIndexInfoSchema() { + return new Schema( + Arrays.asList("table_name", "col_name", "order", "part_num", "root_page_num", "key_schema_typeid", + "key_schema_typesize", "height"), + Arrays.asList(Type.stringType(32), Type.stringType(32), Type.intType(), Type.intType(), + Type.longType(), Type.intType(), Type.intType(), Type.intType()) + ); + } + + // a single row of information_schema.tables + private static class TableInfoRecord { + String tableName; + int partNum; + long pageNum; + boolean isTemporary; + Schema schema; + + TableInfoRecord(String tableName) { + this.tableName = tableName; + this.partNum = -1; + this.pageNum = -1; + this.isTemporary = false; + this.schema = new Schema(Collections.emptyList(), Collections.emptyList()); + } + + TableInfoRecord(Record record) { + List values = record.getValues(); + tableName = values.get(0).getString(); + partNum = values.get(1).getInt(); + pageNum = values.get(2).getLong(); + isTemporary = values.get(3).getBool(); + schema = Schema.fromBytes(ByteBuffer.wrap(values.get(4).toBytes())); + } + + List toDataBox() { + return Arrays.asList( + new StringDataBox(tableName, 32), + new IntDataBox(partNum), + new LongDataBox(pageNum), + new BoolDataBox(isTemporary), + new StringDataBox(new String(schema.toBytes()), MAX_SCHEMA_SIZE) + ); + } + + boolean isAllocated() { + return this.partNum >= 0; + } + } + + // row of information_schema.indices --> BPlusTreeMetadata + private BPlusTreeMetadata parseIndexMetadata(Record record) { + List values = record.getValues(); + String tableName = values.get(0).getString(); + String colName = values.get(1).getString(); + int order = values.get(2).getInt(); + int partNum = values.get(3).getInt(); + long rootPageNum = values.get(4).getLong(); + int height = values.get(7).getInt(); + + if (partNum < 0) { + return null; + } + + Type keySchema = new Type(TypeId.values()[values.get(5).getInt()], values.get(6).getInt()); + return new BPlusTreeMetadata(tableName, colName, keySchema, order, partNum, rootPageNum, height); + } + + // get the lock context for information_schema.tables + private LockContext getTableInfoContext() { + return lockManager.databaseContext().childContext(TABLE_INFO_TABLE_NAME, 1L); + } + + // get the lock context for information_schema.indices + private LockContext getIndexInfoContext() { + return lockManager.databaseContext().childContext(INDEX_INFO_TABLE_NAME, 2L); + } + + // get the lock context for a table + private LockContext getTableContext(String table, int partNum) { + return lockManager.databaseContext().childContext(prefixUserTableName(table), partNum); + } + + // get the lock context for a table + private LockContext getTableContext(String table) { + return getTableContext(table, tableLookup.get(prefixUserTableName(table)).getPartNum()); + } + + // get the lock context for an index + private LockContext getIndexContext(String index, int partNum) { + return lockManager.databaseContext().childContext("indices." + index, partNum); + } + + // get the lock context for an index + LockContext getIndexContext(String index) { + return getIndexContext(index, indexLookup.get(index).getPartNum()); + } + + private String prefixUserTableName(String table) { + if (table.contains(".")) { + return table; + } else { + return "tables." + table; + } + } + + // safely creates a row in information_schema.tables for tableName if none exists + // (with isAllocated=false), and locks the table metadata row with the specified lock to + // ensure no changes can be made until the current transaction commits. + void lockTableMetadata(String tableName, LockType lockType) { + // can't do this in one .compute() call, because we may need to block requesting + // locks on the database/information_schema.tables, and a large part of tableInfoLookup + // will be blocked while we're inside a compute call. + boolean mayNeedToCreate = !tableInfoLookup.containsKey(tableName); + if (mayNeedToCreate) { + // TODO(hw4_part2): acquire all locks needed on database/information_schema.tables before compute() + tableInfoLookup.compute(tableName, (tableName_, recordId) -> { + if (recordId != null) { // record created between containsKey call and this + return recordId; + } + // should not block + return Database.this.tableInfo.addRecord(new TableInfoRecord(tableName_).toDataBox()); + }); + } + + // TODO(hw4_part2): acquire all locks needed on the row in information_schema.tables + } + + private TableInfoRecord getTableMetadata(String tableName) { + RecordId rid = tableInfoLookup.get(tableName); + if (rid == null) { + return new TableInfoRecord(tableName); + } + return new TableInfoRecord(tableInfo.getRecord(rid)); + } + + // safely creates a row in information_schema.indices for tableName,columnName if none exists + // (with partNum=-1), and locks the index metadata row with the specified lock to ensure no + // changes can be made until the current transaction commits. + void lockIndexMetadata(String indexName, LockType lockType) { + // see getTableMetadata - same logic/structure, just with a different table + boolean mayNeedToCreate = !indexInfoLookup.containsKey(indexName); + if (mayNeedToCreate) { + // TODO(hw4_part2): acquire all locks needed on database/information_schema.indices before compute() + indexInfoLookup.compute(indexName, (indexName_, recordId) -> { + if (recordId != null) { // record created between containsKey call and this + return recordId; + } + String[] parts = indexName.split(",", 2); + return Database.this.indexInfo.addRecord(Arrays.asList( + new StringDataBox(parts[0], 32), + new StringDataBox(parts[1], 32), + new IntDataBox(-1), + new IntDataBox(-1), + new LongDataBox(DiskSpaceManager.INVALID_PAGE_NUM), + new IntDataBox(TypeId.INT.ordinal()), + new IntDataBox(4), + new IntDataBox(-1) + )); + }); + } + + // TODO(hw4_part2): acquire all locks needed on the row in information_schema.indices + } + + private BPlusTreeMetadata getIndexMetadata(String tableName, String columnName) { + String indexName = tableName + "," + columnName; + RecordId rid = indexInfoLookup.get(indexName); + if (rid == null) { + return null; + } + return parseIndexMetadata(indexInfo.getRecord(rid)); + } + + /** + * Start a new transaction. + * + * @return the new Transaction + */ + public synchronized Transaction beginTransaction() { + TransactionImpl t = new TransactionImpl(this.numTransactions, false); + activeTransactions.register(); + if (activeTransactions.isTerminated()) { + activeTransactions = new Phaser(1); + } + + this.recoveryManager.startTransaction(t); + ++this.numTransactions; + return t; + } + + /** + * Start a transaction for recovery. + * + * @param transactionNum transaction number + * @return the Transaction + */ + private synchronized Transaction beginRecoveryTranscation(Long transactionNum) { + this.numTransactions = Math.max(this.numTransactions, transactionNum + 1); + + TransactionImpl t = new TransactionImpl(transactionNum, true); + activeTransactions.register(); + if (activeTransactions.isTerminated()) { + activeTransactions = new Phaser(1); + } + + return t; + } + + /** + * Gets the transaction number counter. This is the number of transactions that + * have been created so far, and also the number of the next transaction to be created. + */ + private synchronized long getTransactionCounter() { + return this.numTransactions; + } + + /** + * Updates the transaction number counter. + * @param newTransactionCounter new transaction number counter + */ + private synchronized void setTransactionCounter(long newTransactionCounter) { + this.numTransactions = newTransactionCounter; + } + + private class TransactionContextImpl extends AbstractTransactionContext { + long transNum; + Map aliases; + Map tempTables; + long tempTableCounter; + + private TransactionContextImpl(long tNum) { + this.transNum = tNum; + this.aliases = new HashMap<>(); + this.tempTables = new HashMap<>(); + this.tempTableCounter = 0; + } + + @Override + public long getTransNum() { + return transNum; + } + + @Override + public int getWorkMemSize() { + return Database.this.getWorkMem(); + } + + @Override + public String createTempTable(Schema schema) { + String tempTableName = "tempTable" + tempTableCounter++; + String tableName = prefixTempTableName(tempTableName); + + int partNum = diskSpaceManager.allocPart(); + long pageNum = diskSpaceManager.allocPage(partNum); + RecordId recordId = tableInfo.addRecord(Arrays.asList( + new StringDataBox(tableName, 32), + new IntDataBox(partNum), + new LongDataBox(pageNum), + new BoolDataBox(true), + new StringDataBox(new String(schema.toBytes()), MAX_SCHEMA_SIZE))); + tableInfoLookup.put(tableName, recordId); + + LockContext lockContext = getTableContext(tableName, partNum); + lockContext.disableChildLocks(); + HeapFile heapFile = new PageDirectory(bufferManager, partNum, pageNum, (short) 0, lockContext); + tempTables.put(tempTableName, new Table(tableName, schema, heapFile, lockContext)); + tableLookup.put(tableName, tempTables.get(tempTableName)); + tableIndices.put(tableName, Collections.emptyList()); + + return tempTableName; + } + + private void deleteTempTable(String tempTableName) { + if (!this.tempTables.containsKey(tempTableName)) { + return; + } + + String tableName = prefixTempTableName(tempTableName); + RecordId recordId = tableInfoLookup.remove(tableName); + Record record = tableInfo.deleteRecord(recordId); + TableInfoRecord tableInfoRecord = new TableInfoRecord(record); + bufferManager.freePart(tableInfoRecord.partNum); + tempTables.remove(tempTableName); + tableLookup.remove(tableName); + tableIndices.remove(tableName); + } + + @Override + public void deleteAllTempTables() { + Set keys = new HashSet<>(tempTables.keySet()); + + for (String tableName : keys) { + deleteTempTable(tableName); + } + } + + @Override + public void setAliasMap(Map aliasMap) { + this.aliases = new HashMap<>(aliasMap); + } + + @Override + public void clearAliasMap() { + this.aliases.clear(); + } + + @Override + public boolean indexExists(String tableName, String columnName) { + try { + resolveIndexFromName(tableName, columnName); + } catch (DatabaseException e) { + return false; + } + return true; + } + + @Override + public void updateIndexMetadata(BPlusTreeMetadata metadata) { + indexInfo.updateRecord(Arrays.asList( + new StringDataBox(metadata.getTableName(), 32), + new StringDataBox(metadata.getColName(), 32), + new IntDataBox(metadata.getOrder()), + new IntDataBox(metadata.getPartNum()), + new LongDataBox(metadata.getRootPageNum()), + new IntDataBox(metadata.getKeySchema().getTypeId().ordinal()), + new IntDataBox(metadata.getKeySchema().getSizeInBytes()), + new IntDataBox(metadata.getHeight()) + ), indexInfoLookup.get(metadata.getName())); + } + + @Override + public Iterator sortedScan(String tableName, String columnName) { + // TODO(hw4_part2): scan locking + + Table tab = getTable(tableName); + try { + Pair index = resolveIndexFromName(tableName, columnName); + return new RecordIterator(tab, index.getSecond().scanAll()); + } catch (DatabaseException e1) { + int offset = getTable(tableName).getSchema().getFieldNames().indexOf(columnName); + try { + return new SortOperator(this, tableName, + Comparator.comparing((Record r) -> r.getValues().get(offset))).iterator(); + } catch (QueryPlanException e2) { + throw new DatabaseException(e2); + } + } + } + + @Override + public Iterator sortedScanFrom(String tableName, String columnName, DataBox startValue) { + // TODO(hw4_part2): scan locking + + Table tab = getTable(tableName); + Pair index = resolveIndexFromName(tableName, columnName); + return new RecordIterator(tab, index.getSecond().scanGreaterEqual(startValue)); + } + + @Override + public Iterator lookupKey(String tableName, String columnName, DataBox key) { + Table tab = getTable(tableName); + Pair index = resolveIndexFromName(tableName, columnName); + return new RecordIterator(tab, index.getSecond().scanEqual(key)); + } + + @Override + public BacktrackingIterator getRecordIterator(String tableName) { + return getTable(tableName).iterator(); + } + + @Override + public BacktrackingIterator getPageIterator(String tableName) { + return getTable(tableName).pageIterator(); + } + + @Override + public BacktrackingIterator getBlockIterator(String tableName, Iterator block, + int maxPages) { + return getTable(tableName).blockIterator(block, maxPages); + } + + @Override + public boolean contains(String tableName, String columnName, DataBox key) { + Pair index = resolveIndexFromName(tableName, columnName); + return index.getSecond().get(key).isPresent(); + } + + @Override + public RecordId addRecord(String tableName, List values) { + Table tab = getTable(tableName); + RecordId rid = tab.addRecord(values); + Schema s = tab.getSchema(); + List colNames = s.getFieldNames(); + + for (String indexName : tableIndices.get(tab.getName())) { + String column = indexName.split(",")[1]; + resolveIndexFromName(tableName, column).getSecond().put(values.get(colNames.indexOf(column)), rid); + } + return rid; + } + + @Override + public RecordId deleteRecord(String tableName, RecordId rid) { + Table tab = getTable(tableName); + Schema s = tab.getSchema(); + + Record rec = tab.deleteRecord(rid); + List values = rec.getValues(); + List colNames = s.getFieldNames(); + + for (String indexName : tableIndices.get(tab.getName())) { + String column = indexName.split(",")[1]; + resolveIndexFromName(tableName, column).getSecond().remove(values.get(colNames.indexOf(column))); + } + return rid; + } + + @Override + public Record getRecord(String tableName, RecordId rid) { + return getTable(tableName).getRecord(rid); + } + + @Override + public RecordId updateRecord(String tableName, List values, RecordId rid) { + Table tab = getTable(tableName); + Schema s = tab.getSchema(); + + Record rec = tab.updateRecord(values, rid); + + List oldValues = rec.getValues(); + List colNames = s.getFieldNames(); + + for (String indexName : tableIndices.get(tab.getName())) { + String column = indexName.split(",")[1]; + int i = colNames.indexOf(column); + BPlusTree tree = resolveIndexFromName(tableName, column).getSecond(); + tree.remove(oldValues.get(i)); + tree.put(values.get(i), rid); + } + return rid; + } + + @Override + public void runUpdateRecordWhere(String tableName, String targetColumnName, + UnaryOperator targetValue, + String predColumnName, PredicateOperator predOperator, DataBox predValue) { + Table tab = getTable(tableName); + Iterator recordIds = tab.ridIterator(); + + Schema s = tab.getSchema(); + int uindex = s.getFieldNames().indexOf(targetColumnName); + int pindex = s.getFieldNames().indexOf(predColumnName); + + while(recordIds.hasNext()) { + RecordId curRID = recordIds.next(); + Record cur = getRecord(tableName, curRID); + List recordCopy = new ArrayList<>(cur.getValues()); + + if (predOperator == null || predOperator.evaluate(recordCopy.get(pindex), predValue)) { + recordCopy.set(uindex, targetValue.apply(recordCopy.get(uindex))); + updateRecord(tableName, recordCopy, curRID); + } + } + } + + @Override + public void runDeleteRecordWhere(String tableName, String predColumnName, + PredicateOperator predOperator, DataBox predValue) { + Table tab = getTable(tableName); + Iterator recordIds = tab.ridIterator(); + + Schema s = tab.getSchema(); + int pindex = s.getFieldNames().indexOf(predColumnName); + + while(recordIds.hasNext()) { + RecordId curRID = recordIds.next(); + Record cur = getRecord(tableName, curRID); + List recordCopy = new ArrayList<>(cur.getValues()); + + if (predOperator == null || predOperator.evaluate(recordCopy.get(pindex), predValue)) { + deleteRecord(tableName, curRID); + } + } + } + + @Override + public Schema getSchema(String tableName) { + return getTable(tableName).getSchema(); + } + + @Override + public Schema getFullyQualifiedSchema(String tableName) { + Schema schema = getTable(tableName).getSchema(); + List newColumnNames = new ArrayList<>(); + for (String oldName : schema.getFieldNames()) { + newColumnNames.add(tableName + "." + oldName); + } + + return new Schema(newColumnNames, schema.getFieldTypes()); + } + + @Override + public TableStats getStats(String tableName) { + return getTable(tableName).getStats(); + } + + @Override + public int getNumDataPages(String tableName) { + return getTable(tableName).getNumDataPages(); + } + + @Override + public int getNumEntriesPerPage(String tableName) { + return getTable(tableName).getNumRecordsPerPage(); + } + + @Override + public int getEntrySize(String tableName) { + return getTable(tableName).getSchema().getSizeInBytes(); + } + + @Override + public long getNumRecords(String tableName) { + return getTable(tableName).getNumRecords(); + } + + @Override + public int getTreeOrder(String tableName, String columnName) { + return resolveIndexMetadataFromName(tableName, columnName).getSecond().getOrder(); + } + + @Override + public int getTreeHeight(String tableName, String columnName) { + return resolveIndexMetadataFromName(tableName, columnName).getSecond().getHeight(); + } + + @Override + public void close() { + // TODO(hw4_part2): release locks held by the transaction + return; + } + + @Override + public String toString() { + return "Transaction Context for Transaction " + transNum; + } + + private Pair resolveIndexMetadataFromName(String tableName, + String columnName) { + if (aliases.containsKey(tableName)) { + tableName = aliases.get(tableName); + } + if (columnName.contains(".")) { + String columnPrefix = columnName.split("\\.")[0]; + if (!tableName.equals(columnPrefix)) { + throw new DatabaseException("Column: " + columnName + " is not a column of " + tableName); + } + columnName = columnName.split("\\.")[1]; + } + // remove tables. - index names do not use it + if (tableName.startsWith("tables.")) { + tableName = tableName.substring(tableName.indexOf(".") + 1); + } + String indexName = tableName + "," + columnName; + + // TODO(hw4_part2): add locking + + BPlusTreeMetadata metadata = getIndexMetadata(tableName, columnName); + if (metadata == null) { + throw new DatabaseException("no index with name " + indexName); + } + return new Pair<>(indexName, metadata); + } + + private Pair resolveIndexFromName(String tableName, + String columnName) { + String indexName = resolveIndexMetadataFromName(tableName, columnName).getFirst(); + return new Pair<>(indexName, Database.this.indexLookup.get(indexName)); + } + + @Override + public Table getTable(String tableName) { + if (this.aliases.containsKey(tableName)) { + tableName = this.aliases.get(tableName); + } + + if (this.tempTables.containsKey(tableName)) { + return this.tempTables.get(tableName); + } + + if (!tableName.startsWith(METADATA_TABLE_PREFIX)) { + tableName = prefixUserTableName(tableName); + } + + // TODO(hw4_part2): add locking + + TableInfoRecord record = getTableMetadata(tableName); + if (!record.isAllocated()) { + throw new DatabaseException("no table with name " + tableName); + } + return Database.this.tableLookup.get(tableName); + } + + private String prefixTempTableName(String name) { + String prefix = "temp." + transNum + "-"; + if (name.startsWith(prefix)) { + return name; + } else { + return prefix + name; + } + } + } + + private class TransactionImpl extends AbstractTransaction { + private long transNum; + private boolean recoveryTransaction; + private TransactionContext transactionContext; + + private TransactionImpl(long transNum, boolean recovery) { + this.transNum = transNum; + this.recoveryTransaction = recovery; + this.transactionContext = new TransactionContextImpl(transNum); + } + + @Override + protected void startCommit() { + // TODO(hw5): replace immediate cleanup() call with job (the commented out code) + + transactionContext.deleteAllTempTables(); + + recoveryManager.commit(transNum); + + this.cleanup(); + /* + executor.execute(this::cleanup); + */ + } + + @Override + protected void startRollback() { + executor.execute(() -> { + recoveryManager.abort(transNum); + this.cleanup(); + }); + } + + @Override + public void cleanup() { + if (getStatus() == Status.COMPLETE) { + return; + } + + if (!this.recoveryTransaction) { + recoveryManager.end(transNum); + } + + transactionContext.close(); + activeTransactions.arriveAndDeregister(); + } + + @Override + public long getTransNum() { + return transNum; + } + + @Override + public void createTable(Schema s, String tableName) { + if (tableName.contains(".") && !tableName.startsWith("tables.")) { + throw new IllegalArgumentException("name of new table may not contain '.'"); + } + + String prefixedTableName = prefixUserTableName(tableName); + TransactionContext.setTransaction(transactionContext); + try { + // TODO(hw4_part2): add locking + + lockTableMetadata(prefixedTableName, LockType.NL); + + TableInfoRecord record = getTableMetadata(prefixedTableName); + if (record.isAllocated()) { + throw new DatabaseException("table " + prefixedTableName + " already exists"); + } + + record.partNum = diskSpaceManager.allocPart(); + record.pageNum = diskSpaceManager.allocPage(record.partNum); + record.isTemporary = false; + record.schema = s; + tableInfo.updateRecord(record.toDataBox(), tableInfoLookup.get(prefixedTableName)); + + LockContext tableContext = getTableContext(prefixedTableName, record.partNum); + HeapFile heapFile = new PageDirectory(bufferManager, record.partNum, record.pageNum, + (short) 0, tableContext); + tableLookup.put(prefixedTableName, new Table(prefixedTableName, s, + heapFile, tableContext)); + tableIndices.put(prefixedTableName, new ArrayList<>()); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void dropTable(String tableName) { + if (tableName.contains(".") && !tableName.startsWith("tables.")) { + throw new IllegalArgumentException("name of table may not contain '.': " + tableName); + } + + String prefixedTableName = prefixUserTableName(tableName); + TransactionContext.setTransaction(transactionContext); + try { + // TODO(hw4_part2): add locking + + lockTableMetadata(prefixedTableName, LockType.NL); + + TableInfoRecord record = getTableMetadata(prefixedTableName); + if (!record.isAllocated()) { + throw new DatabaseException("table " + prefixedTableName + " does not exist"); + } + + for (String indexName : new ArrayList<>(tableIndices.get(prefixedTableName))) { + String[] parts = indexName.split(","); + dropIndex(parts[0], parts[1]); + } + + RecordId tableRecordId = tableInfoLookup.get(prefixedTableName); + tableInfo.updateRecord(new TableInfoRecord(prefixedTableName).toDataBox(), tableRecordId); + + tableIndices.remove(prefixedTableName); + tableLookup.remove(prefixedTableName); + bufferManager.freePart(record.partNum); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void dropAllTables() { + TransactionContext.setTransaction(transactionContext); + try { + // TODO(hw4_part2): add locking + + List tableNames = new ArrayList<>(tableLookup.keySet()); + + for (String s : tableNames) { + if (s.startsWith("tables.")) { + this.dropTable(s); + } + } + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void createIndex(String tableName, String columnName, boolean bulkLoad) { + if (tableName.contains(".") && !tableName.startsWith("tables.")) { + throw new IllegalArgumentException("name of table may not contain '.'"); + } + String prefixedTableName = prefixUserTableName(tableName); + TransactionContext.setTransaction(transactionContext); + try { + // TODO(hw4_part2): add locking + + lockTableMetadata(prefixedTableName, LockType.NL); + + TableInfoRecord tableMetadata = getTableMetadata(prefixedTableName); + if (!tableMetadata.isAllocated()) { + throw new DatabaseException("table " + tableName + " does not exist"); + } + + Schema s = tableMetadata.schema; + List schemaColNames = s.getFieldNames(); + List schemaColType = s.getFieldTypes(); + if (!schemaColNames.contains(columnName)) { + throw new DatabaseException("table " + tableName + " does not have a column " + columnName); + } + + int columnIndex = schemaColNames.indexOf(columnName); + Type colType = schemaColType.get(columnIndex); + String indexName = tableName + "," + columnName; + + lockIndexMetadata(indexName, LockType.NL); + + BPlusTreeMetadata metadata = getIndexMetadata(tableName, columnName); + if (metadata != null) { + throw new DatabaseException("index already exists on " + tableName + "(" + columnName + ")"); + } + + int order = BPlusTree.maxOrder(BufferManager.EFFECTIVE_PAGE_SIZE, colType); + List values = Arrays.asList( + new StringDataBox(tableName, 32), + new StringDataBox(columnName, 32), + new IntDataBox(order), + new IntDataBox(diskSpaceManager.allocPart()), + new LongDataBox(DiskSpaceManager.INVALID_PAGE_NUM), + new IntDataBox(colType.getTypeId().ordinal()), + new IntDataBox(colType.getSizeInBytes()), + new IntDataBox(-1) + ); + indexInfo.updateRecord(values, indexInfoLookup.get(indexName)); + metadata = parseIndexMetadata(new Record(values)); + assert (metadata != null); + + LockContext indexContext = getIndexContext(indexName, metadata.getPartNum()); + indexLookup.put(indexName, new BPlusTree(bufferManager, metadata, indexContext)); + tableIndices.get(prefixedTableName).add(indexName); + + // load data into index + Table table = tableLookup.get(prefixedTableName); + BPlusTree tree = indexLookup.get(indexName); + if (bulkLoad) { + throw new UnsupportedOperationException("not implemented"); + } else { + for (RecordId rid : (Iterable) table::ridIterator) { + Record record = table.getRecord(rid); + tree.put(record.getValues().get(columnIndex), rid); + } + } + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void dropIndex(String tableName, String columnName) { + String prefixedTableName = prefixUserTableName(tableName); + String indexName = tableName + "," + columnName; + TransactionContext.setTransaction(transactionContext); + try { + // TODO(hw4_part2): add locking + + lockIndexMetadata(indexName, LockType.NL); + + BPlusTreeMetadata metadata = getIndexMetadata(tableName, columnName); + if (metadata == null) { + throw new DatabaseException("no index on " + tableName + "(" + columnName + ")"); + } + indexInfo.updateRecord(Arrays.asList( + new StringDataBox(tableName, 32), + new StringDataBox(columnName, 32), + new IntDataBox(-1), + new IntDataBox(-1), + new LongDataBox(DiskSpaceManager.INVALID_PAGE_NUM), + new IntDataBox(TypeId.INT.ordinal()), + new IntDataBox(4), + new IntDataBox(-1) + ), indexInfoLookup.get(indexName)); + + bufferManager.freePart(metadata.getPartNum()); + indexLookup.remove(indexName); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public QueryPlan query(String tableName) { + return new QueryPlan(transactionContext, tableName); + } + + @Override + public QueryPlan query(String tableName, String alias) { + return new QueryPlan(transactionContext, tableName, alias); + } + + @Override + public void insert(String tableName, List values) { + TransactionContext.setTransaction(transactionContext); + try { + transactionContext.addRecord(tableName, values); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void update(String tableName, String targetColumnName, UnaryOperator targetValue) { + update(tableName, targetColumnName, targetValue, null, null, null); + } + + @Override + public void update(String tableName, String targetColumnName, UnaryOperator targetValue, + String predColumnName, PredicateOperator predOperator, DataBox predValue) { + TransactionContext.setTransaction(transactionContext); + try { + transactionContext.runUpdateRecordWhere(tableName, targetColumnName, targetValue, predColumnName, + predOperator, predValue); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void delete(String tableName, String predColumnName, PredicateOperator predOperator, + DataBox predValue) { + TransactionContext.setTransaction(transactionContext); + try { + transactionContext.runDeleteRecordWhere(tableName, predColumnName, predOperator, predValue); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void savepoint(String savepointName) { + TransactionContext.setTransaction(transactionContext); + try { + recoveryManager.savepoint(transNum, savepointName); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void rollbackToSavepoint(String savepointName) { + TransactionContext.setTransaction(transactionContext); + try { + recoveryManager.rollbackToSavepoint(transNum, savepointName); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public void releaseSavepoint(String savepointName) { + TransactionContext.setTransaction(transactionContext); + try { + recoveryManager.releaseSavepoint(transNum, savepointName); + } finally { + TransactionContext.unsetTransaction(); + } + } + + @Override + public Schema getSchema(String tableName) { + return transactionContext.getSchema(tableName); + } + + @Override + public Schema getFullyQualifiedSchema(String tableName) { + return transactionContext.getSchema(tableName); + } + + @Override + public TableStats getStats(String tableName) { + return transactionContext.getStats(tableName); + } + + @Override + public int getNumDataPages(String tableName) { + return transactionContext.getNumDataPages(tableName); + } + + @Override + public int getNumEntriesPerPage(String tableName) { + return transactionContext.getNumEntriesPerPage(tableName); + } + + @Override + public int getEntrySize(String tableName) { + return transactionContext.getEntrySize(tableName); + } + + @Override + public long getNumRecords(String tableName) { + return transactionContext.getNumRecords(tableName); + } + + @Override + public int getTreeOrder(String tableName, String columnName) { + return transactionContext.getTreeOrder(tableName, columnName); + } + + @Override + public int getTreeHeight(String tableName, String columnName) { + return transactionContext.getTreeHeight(tableName, columnName); + } + + @Override + public TransactionContext getTransactionContext() { + return transactionContext; + } + + @Override + public String toString() { + return "Transaction " + transNum + " (" + getStatus().toString() + ")"; + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/DatabaseException.java b/src/main/java/edu/berkeley/cs186/database/DatabaseException.java new file mode 100644 index 0000000..9d6631a --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/DatabaseException.java @@ -0,0 +1,11 @@ +package edu.berkeley.cs186.database; + +public class DatabaseException extends RuntimeException { + public DatabaseException(String message) { + super(message); + } + + public DatabaseException(Exception e) { + super(e); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/ThreadPool.java b/src/main/java/edu/berkeley/cs186/database/ThreadPool.java new file mode 100644 index 0000000..f863017 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/ThreadPool.java @@ -0,0 +1,34 @@ +package edu.berkeley.cs186.database; + +import java.util.concurrent.*; + +class ThreadPool extends ThreadPoolExecutor { + ThreadPool() { + super(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>()); + } + + @Override + protected void afterExecute(Runnable r, Throwable t) { + super.afterExecute(r, t); + if (t == null && r instanceof Future) { + try { + ((Future) r).get(); + } catch (CancellationException ce) { + t = ce; + } catch (ExecutionException ee) { + t = ee.getCause(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); // ignore/reset + } + } + if (t != null) { + rethrow(t); + } + } + + @SuppressWarnings("unchecked") + private static void rethrow(Throwable t) throws T { + // rethrows checked exceptions as unchecked + throw (T) t; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/Transaction.java b/src/main/java/edu/berkeley/cs186/database/Transaction.java new file mode 100644 index 0000000..b8352dc --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/Transaction.java @@ -0,0 +1,305 @@ +package edu.berkeley.cs186.database; + +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.query.QueryPlan; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.stats.TableStats; + +import java.util.List; +import java.util.function.UnaryOperator; + +/** + * The public-facing interface of a transaction. + */ +@SuppressWarnings("unused") +public interface Transaction extends AutoCloseable { + // Status /////////////////////////////////////////////////////////////// + enum Status { + RUNNING, + COMMITTING, + ABORTING, + COMPLETE, + RECOVERY_ABORTING; // "ABORTING" state for txns during restart recovery + + private static Status[] values = Status.values(); + + public int getValue() { + return ordinal(); + } + + public static Status fromInt(int x) { + if (x < 0 || x >= values.length) { + String err = String.format("Unknown TypeId ordinal %d.", x); + throw new IllegalArgumentException(err); + } + return values[x]; + } + } + + /** + * @return transaction number + */ + long getTransNum(); + + /** + * @return current status of transaction + */ + Status getStatus(); + + /** + * Sets status of transaction. Should not be used directly by + * users of the transaction (this should be called by the recovery + * manager). + * @param status new status of transaction + */ + void setStatus(Status status); + + /** + * Commits a transaction. Equivalent to + * COMMIT + * + * This is the default way a transaction ends. + */ + void commit(); + + /** + * Rolls back a transaction. Equivalent to + * ROLLBACK + * + * HW5 (Recovery) must be fully implemented. + */ + void rollback(); + + /** + * Cleanup transaction (when transaction ends). Does not + * need to be called directly, as commit/rollback should + * call cleanup themselves. Does not do anything on successive calls + * when called multiple times. + */ + void cleanup(); + + @Override + void close(); + + // DDL ////////////////////////////////////////////////////////////////// + + /** + * Creates a table. Equivalent to + * CREATE TABLE tableName (...s) + * + * Indices must be created afterwards with createIndex. + * + * @param s schema of new table + * @param tableName name of new table + */ + void createTable(Schema s, String tableName); + + /** + * Drops a table. Equivalent to + * DROP TABLE tableName + * + * @param tableName name of table to drop + */ + void dropTable(String tableName); + + /** + * Drops all normal tables. + */ + void dropAllTables(); + + /** + * Creates an index. Equivalent to + * CREATE INDEX tableName_columnName ON tableName (columnName) + * in postgres. + * + * The only index supported is a B+ tree. Indices require HW2 (B+ trees) to + * be fully implemented. Bulk loading requires HW3 Part 1 (Joins/Sorting) to be + * fully implemented as well. + * + * @param tableName name of table to create index for + * @param columnName name of column to create index on + * @param bulkLoad whether to bulk load data + */ + void createIndex(String tableName, String columnName, boolean bulkLoad); + + /** + * Drops an index. Equivalent to + * DROP INDEX tableName_columnName + * in postgres. + * + * @param tableName name of table to drop index from + * @param columnName name of column to drop index from + */ + void dropIndex(String tableName, String columnName); + + // DML ////////////////////////////////////////////////////////////////// + + /** + * Returns a QueryPlan selecting from tableName. Equivalent to + * SELECT * FROM tableName + * and used for all SELECT queries. + * @param tableName name of table to select from + * @return new query plan + */ + QueryPlan query(String tableName); + + /** + * Returns a QueryPlan selecting from tableName. Equivalent to + * SELECT * FROM tableName AS alias + * and used for all SELECT queries. + * @param tableName name of table to select from + * @param alias alias of tableName + * @return new query plan + */ + QueryPlan query(String tableName, String alias); + + /** + * Inserts a row into a table. Equivalent to + * INSERT INTO tableName VALUES(...values) + * + * @param tableName name of table to insert into + * @param values list of values to insert (in the same order as the table's schema) + */ + void insert(String tableName, List values); + + /** + * Updates rows in a table. Equivalent to + * UPDATE tableName SET targetColumnName = targetValue(targetColumnName) + * + * @param tableName name of table to update + * @param targetColumnName column to update + * @param targetValue function mapping old values to new values + */ + void update(String tableName, String targetColumnName, UnaryOperator targetValue); + + /** + * Updates rows in a table. Equivalent to + * UPDATE tableName SET targetColumnName = targetValue(targetColumnName) + * WHERE predColumnName predOperator predValue + * + * @param tableName name of table to update + * @param targetColumnName column to update + * @param targetValue function mapping old values to new values + * @param predColumnName column used in WHERE predicate + * @param predOperator operator used in WHERE predicate + * @param predValue value used in WHERE predicate + */ + void update(String tableName, String targetColumnName, UnaryOperator targetValue, + String predColumnName, PredicateOperator predOperator, DataBox predValue); + + /** + * Deletes rows from a table. Equivalent to + * DELETE FROM tableNAME WHERE predColumnName predOperator predValue + * + * @param tableName name of table to delete from + * @param predColumnName column used in WHERE predicate + * @param predOperator operator used in WHERE predicate + * @param predValue value used in WHERE predicate + */ + void delete(String tableName, String predColumnName, PredicateOperator predOperator, + DataBox predValue); + + // Savepoints /////////////////////////////////////////////////////////// + + /** + * Creates a savepoint. A transaction may roll back to a savepoint it created + * at any point before committing/aborting. Equivalent to + * SAVEPOINT savepointName + * + * Savepoints require HW5 (recovery) to be fully implemented. + * + * @param savepointName name of savepoint + */ + void savepoint(String savepointName); + + /** + * Rolls back all changes made by the transaction since the given savepoint. + * Equivalent to + * ROLLBACK TO savepointName + * + * Savepoints require HW5 (recovery) to be fully implemented. + * + * @param savepointName name of savepoint + */ + void rollbackToSavepoint(String savepointName); + + /** + * Deletes a savepoint. Equivalent to + * RELEASE SAVEPOINT + * + * Savepoints require HW5 (recovery) to be fully implemented. + * + * @param savepointName name of savepoint + */ + void releaseSavepoint(String savepointName); + + // Schema /////////////////////////////////////////////////////////////// + + /** + * @param tableName name of table to get schema of + * @return schema of table + */ + Schema getSchema(String tableName); + + /** + * Same as getSchema, except all column names are fully qualified (tableName.colName). + * + * @param tableName name of table to get schema of + * @return schema of table + */ + Schema getFullyQualifiedSchema(String tableName); + + // Statistics /////////////////////////////////////////////////////////// + + /** + * @param tableName name of table to get stats of + * @return TableStats object of the table + */ + TableStats getStats(String tableName); + + /** + * @param tableName name of table + * @return number of data pages used by the table + */ + int getNumDataPages(String tableName); + + /** + * @param tableName name of table + * @return number of entries that fit on one page for the table + */ + int getNumEntriesPerPage(String tableName); + + /** + * @param tableName name of table + * @return size of a single row for the table + */ + int getEntrySize(String tableName); + + /** + * @param tableName name of table + * @return number of records in the table + */ + long getNumRecords(String tableName); + + /** + * @param tableName name of table + * @param columnName name of column + * @return order of B+ tree index on tableName.columnName + */ + int getTreeOrder(String tableName, String columnName); + + /** + * @param tableName name of table + * @param columnName name of column + * @return height of B+ tree index on tableName.columnName + */ + int getTreeHeight(String tableName, String columnName); + + // Internal ///////////////////////////////////////////////////////////// + + /** + * @return transaction context for this transaction + */ + TransactionContext getTransactionContext(); +} diff --git a/src/main/java/edu/berkeley/cs186/database/TransactionContext.java b/src/main/java/edu/berkeley/cs186/database/TransactionContext.java new file mode 100644 index 0000000..d93bc21 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/TransactionContext.java @@ -0,0 +1,238 @@ +package edu.berkeley.cs186.database; + +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.index.BPlusTreeMetadata; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.RecordId; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.Table; +import edu.berkeley.cs186.database.table.stats.TableStats; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.UnaryOperator; + +/** + * Internal transaction-specific methods, used for implementing parts of the database. + * + * The transaction context for the transaction currently running on the current thread + * can be fetched via TransactionContext::getTransaction; it is only set during the middle + * of a Transaction call. + */ +public interface TransactionContext extends AutoCloseable { + Map> threadTransactions = new ConcurrentHashMap<>(); + + /** + * Fetches the current transaction running on this thread. + * @return transaction actively running on this thread or null if none + */ + static TransactionContext getTransaction() { + long threadId = Thread.currentThread().getId(); + List transactions = threadTransactions.get(threadId); + if (transactions != null && transactions.size() > 0) { + return transactions.get(transactions.size() - 1); + } + return null; + } + + /** + * Sets the current transaction running on this thread. + * @param transaction transaction currently running + */ + static void setTransaction(TransactionContext transaction) { + long threadId = Thread.currentThread().getId(); + threadTransactions.putIfAbsent(threadId, new ArrayList<>()); + threadTransactions.get(threadId).add(transaction); + } + + /** + * Unsets the current transaction running on this thread. + */ + static void unsetTransaction() { + long threadId = Thread.currentThread().getId(); + List transactions = threadTransactions.get(threadId); + if (transactions == null || transactions.size() == 0) { + throw new IllegalStateException("no transaction to unset"); + } + transactions.remove(transactions.size() - 1); + } + + // Status /////////////////////////////////////////////////////////////// + + /** + * @return transaction number + */ + long getTransNum(); + + int getWorkMemSize(); + + @Override + void close(); + + // Temp Tables and Aliasing ///////////////////////////////////////////// + + /** + * Create a temporary table within this transaction. + * + * @param schema the table schema + * @return name of the tempTable + */ + String createTempTable(Schema schema); + + /** + * Deletes all temporary tables within this transaction. + */ + void deleteAllTempTables(); + + /** + * Specify an alias mapping for this transaction. Recursive aliasing is + * not allowed. + * @param aliasMap mapping of alias names to original table names + */ + void setAliasMap(Map aliasMap); + + /** + * Clears any aliases set. + */ + void clearAliasMap(); + + // Indices ////////////////////////////////////////////////////////////// + + /** + * Perform a check to see if the database has an index on this (table,column). + * + * @param tableName the name of the table + * @param columnName the name of the column + * @return boolean if the index exists + */ + boolean indexExists(String tableName, String columnName); + + void updateIndexMetadata(BPlusTreeMetadata metadata); + + // Scans //////////////////////////////////////////////////////////////// + + Iterator sortedScan(String tableName, String columnName); + + Iterator sortedScanFrom(String tableName, String columnName, DataBox startValue); + + Iterator lookupKey(String tableName, String columnName, DataBox key); + + BacktrackingIterator getRecordIterator(String tableName); + + BacktrackingIterator getPageIterator(String tableName); + + BacktrackingIterator getBlockIterator(String tableName, Iterator block, int maxPages); + + boolean contains(String tableName, String columnName, DataBox key); + + // Record Operations ///////////////////////////////////////////////////// + + RecordId addRecord(String tableName, List values); + + RecordId deleteRecord(String tableName, RecordId rid); + + Record getRecord(String tableName, RecordId rid); + + RecordId updateRecord(String tableName, List values, RecordId rid); + + void runUpdateRecordWhere(String tableName, String targetColumnName, + UnaryOperator targetValue, + String predColumnName, PredicateOperator predOperator, DataBox predValue); + + void runDeleteRecordWhere(String tableName, String predColumnName, PredicateOperator predOperator, + DataBox predValue); + + // Table/Schema ///////////////////////////////////////////////////////// + + /** + * @param tableName name of table to get schema of + * @return schema of table + */ + Schema getSchema(String tableName); + + /** + * Same as getSchema, except all column names are fully qualified (tableName.colName). + * + * @param tableName name of table to get schema of + * @return schema of table + */ + Schema getFullyQualifiedSchema(String tableName); + + Table getTable(String tableName); + + // Statistics /////////////////////////////////////////////////////////// + + /** + * @param tableName name of table to get stats of + * @return TableStats object of the table + */ + TableStats getStats(String tableName); + + /** + * @param tableName name of table + * @return number of data pages used by the table + */ + int getNumDataPages(String tableName); + + /** + * @param tableName name of table + * @return number of entries that fit on one page for the table + */ + int getNumEntriesPerPage(String tableName); + + /** + * @param tableName name of table + * @return size of a single row for the table + */ + int getEntrySize(String tableName); + + /** + * @param tableName name of table + * @return number of records in the table + */ + long getNumRecords(String tableName); + + /** + * @param tableName name of table + * @param columnName name of column + * @return order of B+ tree index on tableName.columnName + */ + int getTreeOrder(String tableName, String columnName); + + /** + * @param tableName name of table + * @param columnName name of column + * @return height of B+ tree index on tableName.columnName + */ + int getTreeHeight(String tableName, String columnName); + + // Synchronization ////////////////////////////////////////////////////// + + /** + * prepareBlock acquires the lock backing the condition variable that the transaction + * waits on. Must be called before block(), and is used to ensure that the unblock() call + * corresponding to the following block() call cannot be run before the transaction blocks. + */ + void prepareBlock(); + + /** + * Blocks the transaction (and thread). prepareBlock() must be called first. + */ + void block(); + + /** + * Unblocks the transaction (and thread running the transaction). + */ + void unblock(); + + /** + * @return if the transaction is blocked + */ + boolean getBlocked(); +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/AbstractBuffer.java b/src/main/java/edu/berkeley/cs186/database/common/AbstractBuffer.java new file mode 100644 index 0000000..9b2d529 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/AbstractBuffer.java @@ -0,0 +1,222 @@ +package edu.berkeley.cs186.database.common; + +import java.nio.ByteBuffer; + +/** + * Partial implementation of a Buffer, which funnels all operations + * into the generic get and put operation. + */ +public abstract class AbstractBuffer implements Buffer { + private int pos; + private byte[] bytes; + private ByteBuffer buf; + + protected AbstractBuffer(int pos) { + this.pos = pos; + this.bytes = new byte[8]; + this.buf = ByteBuffer.wrap(this.bytes); + } + + @Override + public abstract Buffer get(byte[] dst, int offset, int length); + + @Override + public final byte get(int index) { + get(bytes, index, 1); + return bytes[0]; + } + + @Override + public final byte get() { + ++this.pos; + return get(this.pos - 1); + } + + @Override + public final Buffer get(byte[] dst) { + pos += dst.length; + return get(dst, pos - dst.length, dst.length); + } + + @Override + public final char getChar() { + ++this.pos; + return getChar(this.pos - 1); + } + + @Override + public final char getChar(int index) { + get(bytes, index, 1); + return buf.getChar(0); + } + + @Override + public final double getDouble() { + this.pos += 8; + return getDouble(this.pos - 8); + } + + @Override + public final double getDouble(int index) { + get(bytes, index, 8); + return buf.getDouble(0); + } + + @Override + public final float getFloat() { + this.pos += 4; + return getFloat(this.pos - 4); + } + + @Override + public final float getFloat(int index) { + get(bytes, index, 4); + return buf.getFloat(0); + } + + @Override + public final int getInt() { + this.pos += 4; + return getInt(this.pos - 4); + } + + @Override + public final int getInt(int index) { + get(bytes, index, 4); + return buf.getInt(0); + } + + @Override + public final long getLong() { + this.pos += 8; + return getLong(this.pos - 8); + } + + @Override + public final long getLong(int index) { + get(bytes, index, 8); + return buf.getLong(0); + } + + @Override + public final short getShort() { + this.pos += 2; + return getShort(this.pos - 2); + } + + @Override + public final short getShort(int index) { + get(bytes, index, 2); + return buf.getShort(0); + } + + @Override + public abstract Buffer put(byte[] src, int offset, int length); + + @Override + public final Buffer put(byte[] src) { + pos += src.length; + return put(src, pos - src.length, src.length); + } + + @Override + public final Buffer put(byte b) { + ++pos; + return put(pos - 1, b); + } + + @Override + public final Buffer put(int index, byte b) { + bytes[0] = b; + return put(bytes, index, 1); + } + + @Override + public final Buffer putChar(char value) { + ++pos; + return putChar(pos - 1, value); + } + + @Override + public final Buffer putChar(int index, char value) { + buf.putChar(0, value); + return put(bytes, index, 1); + } + + @Override + public final Buffer putDouble(double value) { + pos += 8; + return putDouble(pos - 8, value); + } + + @Override + public final Buffer putDouble(int index, double value) { + buf.putDouble(0, value); + return put(bytes, index, 8); + } + + @Override + public final Buffer putFloat(float value) { + pos += 4; + return putFloat(pos - 4, value); + } + + @Override + public final Buffer putFloat(int index, float value) { + buf.putFloat(0, value); + return put(bytes, index, 4); + } + + @Override + public final Buffer putInt(int value) { + pos += 4; + return putInt(pos - 4, value); + } + + @Override + public final Buffer putInt(int index, int value) { + buf.putInt(0, value); + return put(bytes, index, 4); + } + + @Override + public final Buffer putLong(long value) { + pos += 8; + return putLong(pos - 8, value); + } + + @Override + public final Buffer putLong(int index, long value) { + buf.putLong(0, value); + return put(bytes, index, 8); + } + + @Override + public final Buffer putShort(short value) { + pos += 2; + return putShort(pos - 2, value); + } + + @Override + public final Buffer putShort(int index, short value) { + buf.putShort(0, value); + return put(bytes, index, 2); + } + + @Override + public abstract Buffer slice(); + + @Override + public abstract Buffer duplicate(); + + @Override + public final int position() { + return this.pos; + } + + @Override + public final Buffer position(int pos) { + this.pos = pos; + return this; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/Bits.java b/src/main/java/edu/berkeley/cs186/database/common/Bits.java new file mode 100644 index 0000000..617c72d --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/Bits.java @@ -0,0 +1,94 @@ +package edu.berkeley.cs186.database.common; + +import java.util.Arrays; + +public class Bits { + public enum Bit { ZERO, ONE } + + /** + * Get the ith bit of a byte where the 0th bit is the most significant bit + * and the 7th bit is the least significant bit. Some examples: + * + * - getBit(0b10000000, 7) == ZERO + * - getBit(0b10000000, 0) == ONE + * - getBit(0b01000000, 1) == ONE + * - getBit(0b00100000, 1) == ZERO + */ + static Bit getBit(byte b, int i) { + if (i < 0 || i >= 8) { + throw new IllegalArgumentException(String.format("index %d out of bounds", i)); + } + return ((b >> (7 - i)) & 1) == 0 ? Bit.ZERO : Bit.ONE; + } + + /** + * Get the ith bit of a byte array where the 0th bit is the most significat + * bit of the first byte. Some examples: + * + * - getBit(new byte[]{0b10000000, 0b00000000}, 0) == ONE + * - getBit(new byte[]{0b01000000, 0b00000000}, 1) == ONE + * - getBit(new byte[]{0b00000000, 0b00000001}, 15) == ONE + */ + public static Bit getBit(byte[] bytes, int i) { + if (bytes.length == 0 || i < 0 || i >= bytes.length * 8) { + String err = String.format("bytes.length = %d; i = %d.", bytes.length, i); + throw new IllegalArgumentException(err); + } + return getBit(bytes[i / 8], i % 8); + } + + /** + * Set the ith bit of a byte where the 0th bit is the most significant bit + * and the 7th bit is the least significant bit. Some examples: + * + * - setBit(0b00000000, 0, ONE) == 0b10000000 + * - setBit(0b00000000, 1, ONE) == 0b01000000 + * - setBit(0b00000000, 2, ONE) == 0b00100000 + */ + static byte setBit(byte b, int i, Bit bit) { + if (i < 0 || i >= 8) { + throw new IllegalArgumentException(String.format("index %d out of bounds", i)); + } + byte mask = (byte) (1 << (7 - i)); + switch (bit) { + case ZERO: { return (byte) (b & ~mask); } + case ONE: { return (byte) (b | mask); } + default: { throw new IllegalArgumentException("Unreachable code."); } + } + } + + /** + * Set the ith bit of a byte array where the 0th bit is the most significant + * bit of the first byte (arr[0]). An example: + * + * byte[] buf = new bytes[2]; // [0b00000000, 0b00000000] + * setBit(buf, 0, ONE); // [0b10000000, 0b00000000] + * setBit(buf, 1, ONE); // [0b11000000, 0b00000000] + * setBit(buf, 2, ONE); // [0b11100000, 0b00000000] + * setBit(buf, 15, ONE); // [0b11100000, 0b00000001] + */ + public static void setBit(byte[] bytes, int i, Bit bit) { + bytes[i / 8] = setBit(bytes[i / 8], i % 8, bit); + } + + /** + * Counts the number of set bits. For example: + * + * - countBits(0b00001010) == 2 + * - countBits(0b11111101) == 7 + */ + public static int countBits(byte b) { + return Integer.bitCount(b); + } + + /** + * Counts the number of set bits. + */ + public static int countBits(byte[] bytes) { + int count = 0; + for (byte b : bytes) { + count += countBits(b); + } + return count; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/Buffer.java b/src/main/java/edu/berkeley/cs186/database/common/Buffer.java new file mode 100644 index 0000000..71332c2 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/Buffer.java @@ -0,0 +1,40 @@ +package edu.berkeley.cs186.database.common; + +public interface Buffer { + Buffer get(byte[] dst, int offset, int length); + byte get(int index); + byte get(); + Buffer get(byte[] dst); + char getChar(); + char getChar(int index); + double getDouble(); + double getDouble(int index); + float getFloat(); + float getFloat(int index); + int getInt(); + int getInt(int index); + long getLong(); + long getLong(int index); + short getShort(); + short getShort(int index); + Buffer put(byte[] src, int offset, int length); + Buffer put(byte[] src); + Buffer put(byte b); + Buffer put(int index, byte b); + Buffer putChar(char value); + Buffer putChar(int index, char value); + Buffer putDouble(double value); + Buffer putDouble(int index, double value); + Buffer putFloat(float value); + Buffer putFloat(int index, float value); + Buffer putInt(int value); + Buffer putInt(int index, int value); + Buffer putLong(long value); + Buffer putLong(int index, long value); + Buffer putShort(short value); + Buffer putShort(int index, short value); + Buffer slice(); + Buffer duplicate(); + int position(); + Buffer position(int pos); +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/ByteBuffer.java b/src/main/java/edu/berkeley/cs186/database/common/ByteBuffer.java new file mode 100644 index 0000000..4456779 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/ByteBuffer.java @@ -0,0 +1,324 @@ +package edu.berkeley.cs186.database.common; + +import java.nio.*; + +/** + * Wrapper around java.nio.ByteBuffer to implement our Buffer interface. + */ +public class ByteBuffer implements Buffer { + private java.nio.ByteBuffer buf; + + private ByteBuffer(java.nio.ByteBuffer buf) { + this.buf = buf; + } + + public static Buffer allocateDirect(int capacity) { + return new ByteBuffer(java.nio.ByteBuffer.allocateDirect(capacity)); + } + + public static Buffer allocate(int capacity) { + return new ByteBuffer(java.nio.ByteBuffer.allocate(capacity)); + } + + public static Buffer wrap(byte[] array, int offset, int length) { + return new ByteBuffer(java.nio.ByteBuffer.wrap(array, offset, length)); + } + + public static Buffer wrap(byte[] array) { + return new ByteBuffer(java.nio.ByteBuffer.wrap(array)); + } + + @Override + public Buffer slice() { + return new ByteBuffer(buf.slice()); + } + + @Override + public Buffer duplicate() { + return new ByteBuffer(buf.duplicate()); + } + + @Override + public byte get() { + return buf.get(); + } + + @Override + public Buffer put(byte b) { + buf.put(b); + return this; + } + + @Override + public byte get(int index) { + return buf.get(index); + } + + @Override + public Buffer put(int index, byte b) { + buf.put(index, b); + return this; + } + + @Override + public Buffer get(byte[] dst, int offset, int length) { + buf.get(dst, offset, length); + return this; + } + + @Override + public Buffer get(byte[] dst) { + buf.get(dst); + return this; + } + + public Buffer put(java.nio.ByteBuffer src) { + buf.put(src); + return this; + } + + @Override + public Buffer put(byte[] src, int offset, int length) { + buf.put(src, offset, length); + return this; + } + + @Override + public Buffer put(byte[] dst) { + buf.put(dst); + return this; + } + + public boolean hasArray() { + return buf.hasArray(); + } + + public byte[] array() { + return buf.array(); + } + + public int arrayOffset() { + return buf.arrayOffset(); + } + + public java.nio.ByteBuffer compact() { + return buf.compact(); + } + + public boolean isDirect() { + return buf.isDirect(); + } + + @Override + public String toString() { + return buf.toString(); + } + + @Override + public int hashCode() { + return buf.hashCode(); + } + + @Override + public boolean equals(Object ob) { + return buf.equals(ob); + } + + public ByteOrder order() { + return buf.order(); + } + + public Buffer order(ByteOrder bo) { + buf.order(bo); + return this; + } + + @Override + public char getChar() { + return buf.getChar(); + } + + @Override + public Buffer putChar(char value) { + buf.putChar(value); + return this; + } + + @Override + public char getChar(int index) { + return buf.getChar(index); + } + + @Override + public Buffer putChar(int index, char value) { + buf.putChar(index, value); + return this; + } + + @Override + public short getShort() { + return buf.getShort(); + } + + @Override + public Buffer putShort(short value) { + buf.putShort(value); + return this; + } + + @Override + public short getShort(int index) { + return buf.getShort(index); + } + + @Override + public Buffer putShort(int index, short value) { + buf.putShort(index, value); + return this; + } + + @Override + public int getInt() { + return buf.getInt(); + } + + @Override + public Buffer putInt(int value) { + buf.putInt(value); + return this; + } + + @Override + public int getInt(int index) { + return buf.getInt(index); + } + + @Override + public Buffer putInt(int index, int value) { + buf.putInt(index, value); + return this; + } + + @Override + public long getLong() { + return buf.getLong(); + } + + @Override + public Buffer putLong(long value) { + buf.putLong(value); + return this; + } + + @Override + public long getLong(int index) { + return buf.getLong(index); + } + + @Override + public Buffer putLong(int index, long value) { + buf.putLong(index, value); + return this; + } + + @Override + public float getFloat() { + return buf.getFloat(); + } + + @Override + public Buffer putFloat(float value) { + buf.putFloat(value); + return this; + } + + @Override + public float getFloat(int index) { + return buf.getFloat(index); + } + + @Override + public Buffer putFloat(int index, float value) { + buf.putFloat(index, value); + return this; + } + + @Override + public double getDouble() { + return buf.getDouble(); + } + + @Override + public Buffer putDouble(double value) { + buf.putDouble(value); + return this; + } + + @Override + public double getDouble(int index) { + return buf.getDouble(index); + } + + @Override + public Buffer putDouble(int index, double value) { + buf.putDouble(index, value); + return this; + } + + public int capacity() { + return buf.capacity(); + } + + public int limit() { + return buf.limit(); + } + + public Buffer limit(int newLimit) { + buf.limit(newLimit); + return this; + } + + public Buffer mark() { + buf.mark(); + return this; + } + + public Buffer reset() { + buf.reset(); + return this; + } + + public Buffer clear() { + buf.clear(); + return this; + } + + public Buffer flip() { + buf.flip(); + return this; + } + + public Buffer rewind() { + buf.rewind(); + return this; + } + + public int remaining() { + return buf.remaining(); + } + + public boolean hasRemaining() { + return buf.hasRemaining(); + } + + @Override + public int position() { + return buf.position(); + } + + @Override + public Buffer position(int pos) { + buf.position(pos); + return this; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/Pair.java b/src/main/java/edu/berkeley/cs186/database/common/Pair.java new file mode 100644 index 0000000..6a991a9 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/Pair.java @@ -0,0 +1,48 @@ +package edu.berkeley.cs186.database.common; + +/** A simple, immutable, generic pair. */ +public class Pair { + private final A first; + private final B second; + + public Pair(A first, B second) { + this.first = first; + this.second = second; + } + + public A getFirst() { + return first; + } + + public B getSecond() { + return second; + } + + @Override + public int hashCode() { + int hashFirst = first != null ? first.hashCode() : 0; + int hashSecond = second != null ? second.hashCode() : 0; + return (hashFirst + hashSecond) * hashSecond + hashFirst; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Pair)) { + return false; + } + + Pair p = (Pair) other; + boolean firstEquals = getFirst() == null + ? p.getFirst() == null + : getFirst().equals(p.getFirst()); + boolean secondEquals = getSecond() == null + ? p.getSecond() == null + : getSecond().equals(p.getSecond()); + return firstEquals && secondEquals; + } + + @Override + public String toString() { + return "(" + first + ", " + second + ")"; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/PredicateOperator.java b/src/main/java/edu/berkeley/cs186/database/common/PredicateOperator.java new file mode 100644 index 0000000..f1a1460 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/PredicateOperator.java @@ -0,0 +1,28 @@ +package edu.berkeley.cs186.database.common; + +public enum PredicateOperator { + EQUALS, + NOT_EQUALS, + LESS_THAN, + LESS_THAN_EQUALS, + GREATER_THAN, + GREATER_THAN_EQUALS; + + public > boolean evaluate(T a, T b) { + switch (this) { + case EQUALS: + return a.compareTo(b) == 0; + case NOT_EQUALS: + return a.compareTo(b) != 0; + case LESS_THAN: + return a.compareTo(b) < 0; + case LESS_THAN_EQUALS: + return a.compareTo(b) <= 0; + case GREATER_THAN: + return a.compareTo(b) > 0; + case GREATER_THAN_EQUALS: + return a.compareTo(b) >= 0; + } + return false; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/iterator/ArrayBacktrackingIterator.java b/src/main/java/edu/berkeley/cs186/database/common/iterator/ArrayBacktrackingIterator.java new file mode 100644 index 0000000..c104915 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/iterator/ArrayBacktrackingIterator.java @@ -0,0 +1,24 @@ +package edu.berkeley.cs186.database.common.iterator; + +/** + * Backtracking iterator over an array. + */ +public class ArrayBacktrackingIterator extends IndexBacktrackingIterator { + protected T[] array; + + public ArrayBacktrackingIterator(T[] array) { + super(array.length); + this.array = array; + } + + @Override + protected int getNextNonempty(int currentIndex) { + return currentIndex + 1; + } + + @Override + protected T getValue(int index) { + return this.array[index]; + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/common/iterator/BacktrackingIterable.java b/src/main/java/edu/berkeley/cs186/database/common/iterator/BacktrackingIterable.java new file mode 100644 index 0000000..0676435 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/iterator/BacktrackingIterable.java @@ -0,0 +1,6 @@ +package edu.berkeley.cs186.database.common.iterator; + +public interface BacktrackingIterable extends Iterable { + @Override + BacktrackingIterator iterator(); +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/iterator/BacktrackingIterator.java b/src/main/java/edu/berkeley/cs186/database/common/iterator/BacktrackingIterator.java new file mode 100644 index 0000000..c5481a9 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/iterator/BacktrackingIterator.java @@ -0,0 +1,35 @@ +package edu.berkeley.cs186.database.common.iterator; + +import java.util.Iterator; + +public interface BacktrackingIterator extends Iterator { + /** + * markPrev() marks the last returned value of the iterator, which is the last + * returned value of next(). + * + * Calling markPrev() on an iterator that has not yielded a record yet, + * or that has not yielded a record since the last reset() call does nothing. + */ + void markPrev(); + + /** + * markNext() marks the next returned value of the iterator, which is the + * value returned by the next call of next(). + * + * Calling markNext() on an iterator that has no records left, + * or that has not yielded a record since the last reset() call does nothing. + */ + void markNext(); + + /** + * reset() resets the iterator to the last marked location. + * + * The next next() call should return the value that was marked - if markPrev() + * was used, this is the value returned by the next() call before markPrev(), and if + * markNext() was used, this is the value returned by the next() call after markNext(). + * If neither mark methods were called, reset() does nothing. You may reset() to the same + * point as many times as desired, as long as neither mark method is called again. + */ + void reset(); +} + diff --git a/src/main/java/edu/berkeley/cs186/database/common/iterator/ConcatBacktrackingIterator.java b/src/main/java/edu/berkeley/cs186/database/common/iterator/ConcatBacktrackingIterator.java new file mode 100644 index 0000000..0443778 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/iterator/ConcatBacktrackingIterator.java @@ -0,0 +1,84 @@ +package edu.berkeley.cs186.database.common.iterator; + +/** + * Iterator that concatenates a bunch of backtracking iterables together. + */ +public class ConcatBacktrackingIterator implements BacktrackingIterator { + private BacktrackingIterator> outerIterator; + + private BacktrackingIterator prevItemIterator; + private BacktrackingIterator nextItemIterator; + private BacktrackingIterator markItemIterator; + + private boolean markMidIterator; + + public ConcatBacktrackingIterator(BacktrackingIterable> outerIterable) { + this(outerIterable.iterator()); + } + + public ConcatBacktrackingIterator(BacktrackingIterator> outerIterator) { + this.outerIterator = outerIterator; + this.prevItemIterator = null; + this.nextItemIterator = new EmptyBacktrackingIterator<>(); + this.markItemIterator = null; + this.markMidIterator = false; + + this.moveNextToNonempty(); + } + + private void moveNextToNonempty() { + while (!this.nextItemIterator.hasNext() && this.outerIterator.hasNext()) { + this.nextItemIterator = this.outerIterator.next().iterator(); + } + } + + @Override + public boolean hasNext() { + return this.nextItemIterator.hasNext(); + } + + @Override + public T next() { + T item = this.nextItemIterator.next(); + this.prevItemIterator = this.nextItemIterator; + this.moveNextToNonempty(); + return item; + } + + @Override + public void markPrev() { + if (this.prevItemIterator == null) { + return; + } + this.markItemIterator = this.prevItemIterator; + this.markItemIterator.markPrev(); + this.outerIterator.markPrev(); + this.markMidIterator = (this.prevItemIterator == this.nextItemIterator); + } + + @Override + public void markNext() { + this.markItemIterator = this.nextItemIterator; + this.markItemIterator.markNext(); + this.outerIterator.markNext(); + this.markMidIterator = false; + } + + @Override + public void reset() { + if (this.markItemIterator == null) { + return; + } + this.prevItemIterator = null; + this.nextItemIterator = this.markItemIterator; + this.nextItemIterator.reset(); + + this.outerIterator.reset(); + // either next = prev at time of markPrev, or next = nonempty after prev + // we want outerIterator.next() to return iterable after prev; it returns next + // skipping the empty iterables after prev is also ok -- since they would be skipped anyways + if (this.markMidIterator) { + this.outerIterator.next(); + } + } +} \ No newline at end of file diff --git a/src/main/java/edu/berkeley/cs186/database/common/iterator/EmptyBacktrackingIterator.java b/src/main/java/edu/berkeley/cs186/database/common/iterator/EmptyBacktrackingIterator.java new file mode 100644 index 0000000..912bb45 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/iterator/EmptyBacktrackingIterator.java @@ -0,0 +1,14 @@ +package edu.berkeley.cs186.database.common.iterator; + +import java.util.NoSuchElementException; + +/** + * Empty backtracking iterator. + */ +public class EmptyBacktrackingIterator implements BacktrackingIterator { + @Override public boolean hasNext() { return false; } + @Override public T next() { throw new NoSuchElementException(); } + @Override public void markPrev() {} + @Override public void markNext() {} + @Override public void reset() {} +} diff --git a/src/main/java/edu/berkeley/cs186/database/common/iterator/IndexBacktrackingIterator.java b/src/main/java/edu/berkeley/cs186/database/common/iterator/IndexBacktrackingIterator.java new file mode 100644 index 0000000..d7479d9 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/common/iterator/IndexBacktrackingIterator.java @@ -0,0 +1,78 @@ +package edu.berkeley.cs186.database.common.iterator; + +import java.util.NoSuchElementException; + +/** + * Partial implementation of a backtracking iterator over an indexable collection + * with some indices possibly not matching to a value. + */ +public abstract class IndexBacktrackingIterator implements BacktrackingIterator { + private int maxIndex; + private int prevIndex = -1; + private int nextIndex = -1; + private int markedIndex = -1; + private int firstIndex = -1; + + public IndexBacktrackingIterator(int maxIndex) { + this.maxIndex = maxIndex; + } + + /** + * Get the next nonempty index. Initial call uses -1. + * @return next nonempty index or the max index if no more values. + */ + protected abstract int getNextNonempty(int currentIndex); + + /** + * Get the value at the given index. Index will always be a value returned + * by getNextNonempty. + * @param index index to get value at + * @return value at index + */ + protected abstract T getValue(int index); + + @Override + public boolean hasNext() { + if (this.nextIndex == -1) { + this.nextIndex = this.firstIndex = this.getNextNonempty(-1); + } + return this.nextIndex < this.maxIndex; + } + + @Override + public T next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + T value = getValue(this.nextIndex); + this.prevIndex = this.nextIndex; + this.nextIndex = this.getNextNonempty(this.nextIndex); + return value; + } + + @Override + public void markPrev() { + // The second condition prevents using mark/reset/mark/reset/.. to + // move the iterator backwards. + if (this.nextIndex <= this.firstIndex || this.prevIndex < this.markedIndex) { + return; + } + this.markedIndex = this.prevIndex; + } + + @Override + public void markNext() { + if (this.nextIndex >= this.maxIndex) { + return; + } + this.markedIndex = this.nextIndex; + } + + @Override + public void reset() { + if (this.markedIndex == -1) { + return; + } + this.nextIndex = this.markedIndex; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/DummyLockContext.java b/src/main/java/edu/berkeley/cs186/database/concurrency/DummyLockContext.java new file mode 100644 index 0000000..6901dbe --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/DummyLockContext.java @@ -0,0 +1,76 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Pair; + +/** + * A lock context that doesn't do anything at all. Used where a lock context + * is expected, but no locking should be done. + */ +public class DummyLockContext extends LockContext { + public DummyLockContext() { + this((LockContext) null); + } + + public DummyLockContext(LockContext parent) { + super(new DummyLockManager(), parent, new Pair<>("Unnamed", -1L)); + } + + public DummyLockContext(Pair name) { + this(null, name); + } + + public DummyLockContext(LockContext parent, Pair name) { + super(new DummyLockManager(), parent, name); + } + + @Override + public void acquire(TransactionContext transaction, LockType lockType) { } + + @Override + public void release(TransactionContext transaction) { } + + @Override + public void promote(TransactionContext transaction, LockType newLockType) { } + + @Override + public void escalate(TransactionContext transaction) { } + + @Override + public void disableChildLocks() { } + + @Override + public LockContext childContext(String readable, long name) { + return new DummyLockContext(this, new Pair<>(readable, name)); + } + + @Override + public int capacity() { + return 0; + } + + @Override + public void capacity(int capacity) { + } + + @Override + public double saturation(TransactionContext transaction) { + return 0.0; + } + + @Override + public LockType getExplicitLockType(TransactionContext transaction) { + return LockType.NL; + } + + @Override + public LockType getEffectiveLockType(TransactionContext transaction) { + return LockType.NL; + } + + @Override + public String toString() { + return "Dummy Lock Context(\" + name.toString() + \")"; + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/DummyLockManager.java b/src/main/java/edu/berkeley/cs186/database/concurrency/DummyLockManager.java new file mode 100644 index 0000000..fe1fc3b --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/DummyLockManager.java @@ -0,0 +1,60 @@ +package edu.berkeley.cs186.database.concurrency; + +import java.util.Collections; +import java.util.List; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Pair; + +/** + * Dummy lock manager that does no locking or error checking. + * + * Used for non-locking-related tests to disable locking. + */ +public class DummyLockManager extends LockManager { + public DummyLockManager() { } + + @Override + public LockContext context(String readable, long name) { + return new DummyLockContext(new Pair<>(readable, name)); + } + + @Override + public LockContext databaseContext() { + return new DummyLockContext(new Pair<>("database", 0L)); + } + + @Override + public void acquireAndRelease(TransactionContext transaction, ResourceName name, + LockType lockType, List releaseLocks) + throws DuplicateLockRequestException, NoLockHeldException { } + + @Override + public void acquire(TransactionContext transaction, ResourceName name, + LockType lockType) throws DuplicateLockRequestException { } + + @Override + public void release(TransactionContext transaction, ResourceName name) + throws NoLockHeldException { } + + @Override + public void promote(TransactionContext transaction, ResourceName name, + LockType newLockType) + throws DuplicateLockRequestException, NoLockHeldException, InvalidLockException { } + + @Override + public LockType getLockType(TransactionContext transaction, ResourceName name) { + return LockType.NL; + } + + @Override + public List getLocks(ResourceName name) { + return Collections.emptyList(); + } + + @Override + public List getLocks(TransactionContext transaction) { + return Collections.emptyList(); + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/DuplicateLockRequestException.java b/src/main/java/edu/berkeley/cs186/database/concurrency/DuplicateLockRequestException.java new file mode 100644 index 0000000..e6b3a01 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/DuplicateLockRequestException.java @@ -0,0 +1,8 @@ +package edu.berkeley.cs186.database.concurrency; + +public class DuplicateLockRequestException extends RuntimeException { + DuplicateLockRequestException(String message) { + super(message); + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/InvalidLockException.java b/src/main/java/edu/berkeley/cs186/database/concurrency/InvalidLockException.java new file mode 100644 index 0000000..a7d570d --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/InvalidLockException.java @@ -0,0 +1,8 @@ +package edu.berkeley.cs186.database.concurrency; + +public class InvalidLockException extends RuntimeException { + InvalidLockException(String message) { + super(message); + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/Lock.java b/src/main/java/edu/berkeley/cs186/database/concurrency/Lock.java new file mode 100644 index 0000000..6549739 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/Lock.java @@ -0,0 +1,36 @@ +package edu.berkeley.cs186.database.concurrency; + +/** + * Represents a lock held by a transaction on a resource. + */ +public class Lock { + public ResourceName name; + public LockType lockType; + public Long transactionNum; + + public Lock(ResourceName name, LockType lockType, long transactionNum) { + this.name = name; + this.lockType = lockType; + this.transactionNum = transactionNum; + } + + @Override + public boolean equals(Object other) { + if (other instanceof Lock) { + Lock l = (Lock) other; + return l.name.equals(name) && lockType == l.lockType && transactionNum.equals(l.transactionNum); + } else { + return false; + } + } + + @Override + public int hashCode() { + return 37 * (37 * name.hashCode() + lockType.hashCode()) + transactionNum.hashCode(); + } + + @Override + public String toString() { + return "T" + transactionNum.toString() + ": " + lockType.toString() + "(" + name.toString() + ")"; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/LockContext.java b/src/main/java/edu/berkeley/cs186/database/concurrency/LockContext.java new file mode 100644 index 0000000..73793ab --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/LockContext.java @@ -0,0 +1,264 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Pair; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * LockContext wraps around LockManager to provide the hierarchical structure + * of multigranularity locking. Calls to acquire/release/etc. locks should + * be mostly done through a LockContext, which provides access to locking + * methods at a certain point in the hierarchy (database, table X, etc.) + */ +public class LockContext { + // You should not remove any of these fields. You may add additional fields/methods as you see fit. + + // The underlying lock manager. + protected final LockManager lockman; + // The parent LockContext object, or null if this LockContext is at the top of the hierarchy. + protected final LockContext parent; + // The name of the resource this LockContext represents. + protected ResourceName name; + // Whether this LockContext is readonly. If a LockContext is readonly, acquire/release/promote/escalate should + // throw an UnsupportedOperationException. + protected boolean readonly; + // A mapping between transaction numbers, and the number of locks on children of this LockContext + // that the transaction holds. + protected final Map numChildLocks; + // The number of children that this LockContext has, if it differs from the number of times + // LockContext#childContext was called with unique parameters: for a table, we do not + // explicitly create a LockContext for every page (we create them as needed), but + // the capacity should be the number of pages in the table, so we use this + // field to override the return value for capacity(). + protected int capacity; + + // You should not modify or use this directly. + protected final Map children; + + // Whether or not any new child LockContexts should be marked readonly. + protected boolean childLocksDisabled; + + public LockContext(LockManager lockman, LockContext parent, Pair name) { + this(lockman, parent, name, false); + } + + protected LockContext(LockManager lockman, LockContext parent, Pair name, + boolean readonly) { + this.lockman = lockman; + this.parent = parent; + if (parent == null) { + this.name = new ResourceName(name); + } else { + this.name = new ResourceName(parent.getResourceName(), name); + } + this.readonly = readonly; + this.numChildLocks = new ConcurrentHashMap<>(); + this.capacity = -1; + this.children = new ConcurrentHashMap<>(); + this.childLocksDisabled = readonly; + } + + /** + * Gets a lock context corresponding to NAME from a lock manager. + */ + public static LockContext fromResourceName(LockManager lockman, ResourceName name) { + Iterator> names = name.getNames().iterator(); + LockContext ctx; + Pair n1 = names.next(); + ctx = lockman.context(n1.getFirst(), n1.getSecond()); + while (names.hasNext()) { + Pair p = names.next(); + ctx = ctx.childContext(p.getFirst(), p.getSecond()); + } + return ctx; + } + + /** + * Get the name of the resource that this lock context pertains to. + */ + public ResourceName getResourceName() { + return name; + } + + /** + * Acquire a LOCKTYPE lock, for transaction TRANSACTION. + * + * Note: you *must* make any necessary updates to numChildLocks, or + * else calls to LockContext#saturation will not work properly. + * + * @throws InvalidLockException if the request is invalid + * @throws DuplicateLockRequestException if a lock is already held by TRANSACTION + * @throws UnsupportedOperationException if context is readonly + */ + public void acquire(TransactionContext transaction, LockType lockType) + throws InvalidLockException, DuplicateLockRequestException { + // TODO(hw4_part1): implement + + return; + } + + /** + * Release TRANSACTION's lock on NAME. + * + * Note: you *must* make any necessary updates to numChildLocks, or + * else calls to LockContext#saturation will not work properly. + * + * @throws NoLockHeldException if no lock on NAME is held by TRANSACTION + * @throws InvalidLockException if the lock cannot be released (because doing so would + * violate multigranularity locking constraints) + * @throws UnsupportedOperationException if context is readonly + */ + public void release(TransactionContext transaction) + throws NoLockHeldException, InvalidLockException { + // TODO(hw4_part1): implement + + return; + } + + /** + * Promote TRANSACTION's lock to NEWLOCKTYPE. For promotion to SIX from IS/IX/S, all S, + * IS, and SIX locks on descendants must be simultaneously released. + * + * Note: you *must* make any necessary updates to numChildLocks, or + * else calls to LockContext#saturation will not work properly. + * + * @throws DuplicateLockRequestException if TRANSACTION already has a NEWLOCKTYPE lock + * @throws NoLockHeldException if TRANSACTION has no lock + * @throws InvalidLockException if the requested lock type is not a promotion or promoting + * would cause the lock manager to enter an invalid state (e.g. IS(parent), X(child)). A promotion + * from lock type A to lock type B is valid if B is substitutable + * for A and B is not equal to A, or if B is SIX and A is IS/IX/S, and invalid otherwise. + * @throws UnsupportedOperationException if context is readonly + */ + public void promote(TransactionContext transaction, LockType newLockType) + throws DuplicateLockRequestException, NoLockHeldException, InvalidLockException { + // TODO(hw4_part1): implement + + return; + } + + /** + * Escalate TRANSACTION's lock from descendants of this context to this level, using either + * an S or X lock. There should be no descendant locks after this + * call, and every operation valid on descendants of this context before this call + * must still be valid. You should only make *one* mutating call to the lock manager, + * and should only request information about TRANSACTION from the lock manager. + * + * For example, if a transaction has the following locks: + * IX(database) IX(table1) S(table2) S(table1 page3) X(table1 page5) + * then after table1Context.escalate(transaction) is called, we should have: + * IX(database) X(table1) S(table2) + * + * You should not make any mutating calls if the locks held by the transaction do not change + * (such as when you call escalate multiple times in a row). + * + * Note: you *must* make any necessary updates to numChildLocks of all relevant contexts, or + * else calls to LockContext#saturation will not work properly. + * + * @throws NoLockHeldException if TRANSACTION has no lock at this level + * @throws UnsupportedOperationException if context is readonly + */ + public void escalate(TransactionContext transaction) throws NoLockHeldException { + // TODO(hw4_part1): implement + + return; + } + + /** + * Gets the type of lock that the transaction has at this level, either implicitly + * (e.g. explicit S lock at higher level implies S lock at this level) or explicitly. + * Returns NL if there is no explicit nor implicit lock. + */ + public LockType getEffectiveLockType(TransactionContext transaction) { + if (transaction == null) { + return LockType.NL; + } + // TODO(hw4_part1): implement + return LockType.NL; + } + + /** + * Get the type of lock that TRANSACTION holds at this level, or NL if no lock is held at this level. + */ + public LockType getExplicitLockType(TransactionContext transaction) { + if (transaction == null) { + return LockType.NL; + } + // TODO(hw4_part1): implement + return LockType.NL; + } + + /** + * Disables locking descendants. This causes all new child contexts of this context + * to be readonly. This is used for indices and temporary tables (where + * we disallow finer-grain locks), the former due to complexity locking + * B+ trees, and the latter due to the fact that temporary tables are only + * accessible to one transaction, so finer-grain locks make no sense. + */ + public void disableChildLocks() { + this.childLocksDisabled = true; + } + + /** + * Gets the parent context. + */ + public LockContext parentContext() { + return parent; + } + + /** + * Gets the context for the child with name NAME (with a readable version READABLE). + */ + public synchronized LockContext childContext(String readable, long name) { + LockContext temp = new LockContext(lockman, this, new Pair<>(readable, name), + this.childLocksDisabled || this.readonly); + LockContext child = this.children.putIfAbsent(name, temp); + if (child == null) { + child = temp; + } + if (child.name.getCurrentName().getFirst() == null && readable != null) { + child.name = new ResourceName(this.name, new Pair<>(readable, name)); + } + return child; + } + + /** + * Gets the context for the child with name NAME. + */ + public synchronized LockContext childContext(long name) { + return childContext(Long.toString(name), name); + } + + /** + * Sets the capacity (number of children). + */ + public synchronized void capacity(int capacity) { + this.capacity = capacity; + } + + /** + * Gets the capacity. Defaults to number of child contexts if never explicitly set. + */ + public synchronized int capacity() { + return this.capacity < 0 ? this.children.size() : this.capacity; + } + + /** + * Gets the saturation (number of locks held on children / number of children) for + * a single transaction. Saturation is 0 if number of children is 0. + */ + public double saturation(TransactionContext transaction) { + if (transaction == null || capacity() == 0) { + return 0.0; + } + return ((double) numChildLocks.getOrDefault(transaction.getTransNum(), 0)) / capacity(); + } + + @Override + public String toString() { + return "LockContext(" + name.toString() + ")"; + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/LockManager.java b/src/main/java/edu/berkeley/cs186/database/concurrency/LockManager.java new file mode 100644 index 0000000..ddf3c5f --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/LockManager.java @@ -0,0 +1,237 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Pair; + +import java.util.*; + +/** + * LockManager maintains the bookkeeping for what transactions have + * what locks on what resources. The lock manager should generally **not** + * be used directly: instead, code should call methods of LockContext to + * acquire/release/promote/escalate locks. + * + * The LockManager is primarily concerned with the mappings between + * transactions, resources, and locks, and does not concern itself with + * multiple levels of granularity (you can and should treat ResourceName + * as a generic Object, rather than as an object encapsulating levels of + * granularity, in this class). + * + * It follows that LockManager should allow **all** + * requests that are valid from the perspective of treating every resource + * as independent objects, even if they would be invalid from a + * multigranularity locking perspective. For example, if LockManager#acquire + * is called asking for an X lock on Table A, and the transaction has no + * locks at the time, the request is considered valid (because the only problem + * with such a request would be that the transaction does not have the appropriate + * intent locks, but that is a multigranularity concern). + * + * Each resource the lock manager manages has its own queue of LockRequest objects + * representing a request to acquire (or promote/acquire-and-release) a lock that + * could not be satisfied at the time. This queue should be processed every time + * a lock on that resource gets released, starting from the first request, and going + * in order until a request cannot be satisfied. Requests taken off the queue should + * be treated as if that transaction had made the request right after the resource was + * released in absence of a queue (i.e. removing a request by T1 to acquire X(db) should + * be treated as if T1 had just requested X(db) and there were no queue on db: T1 should + * be given the X lock on db, and put in an unblocked state via Transaction#unblock). + * + * This does mean that in the case of: + * queue: S(A) X(A) S(A) + * only the first request should be removed from the queue when the queue is processed. + */ +public class LockManager { + // transactionLocks is a mapping from transaction number to a list of lock + // objects held by that transaction. + private Map> transactionLocks = new HashMap<>(); + // resourceEntries is a mapping from resource names to a ResourceEntry + // object, which contains a list of Locks on the object, as well as a + // queue for requests on that resource. + private Map resourceEntries = new HashMap<>(); + + // A ResourceEntry contains the list of locks on a resource, as well as + // the queue for requests for locks on the resource. + private class ResourceEntry { + // List of currently granted locks on the resource. + List locks = new ArrayList<>(); + // Queue for yet-to-be-satisfied lock requests on this resource. + Deque waitingQueue = new ArrayDeque<>(); + + // TODO(hw4_part1): You may add helper methods here if you wish + + @Override + public String toString() { + return "Active Locks: " + Arrays.toString(this.locks.toArray()) + + ", Queue: " + Arrays.toString(this.waitingQueue.toArray()); + } + } + + // You should not modify or use this directly. + private Map contexts = new HashMap<>(); + + /** + * Helper method to fetch the resourceEntry corresponding to NAME. + * Inserts a new (empty) resourceEntry into the map if no entry exists yet. + */ + private ResourceEntry getResourceEntry(ResourceName name) { + resourceEntries.putIfAbsent(name, new ResourceEntry()); + return resourceEntries.get(name); + } + + // TODO(hw4_part1): You may add helper methods here if you wish + + /** + * Acquire a LOCKTYPE lock on NAME, for transaction TRANSACTION, and releases all locks + * in RELEASELOCKS after acquiring the lock, in one atomic action. + * + * Error checking must be done before any locks are acquired or released. If the new lock + * is not compatible with another transaction's lock on the resource, the transaction is + * blocked and the request is placed at the **front** of ITEM's queue. + * + * Locks in RELEASELOCKS should be released only after the requested lock has been acquired. + * The corresponding queues should be processed. + * + * An acquire-and-release that releases an old lock on NAME **should not** change the + * acquisition time of the lock on NAME, i.e. + * if a transaction acquired locks in the order: S(A), X(B), acquire X(A) and release S(A), the + * lock on A is considered to have been acquired before the lock on B. + * + * @throws DuplicateLockRequestException if a lock on NAME is held by TRANSACTION and + * isn't being released + * @throws NoLockHeldException if no lock on a name in RELEASELOCKS is held by TRANSACTION + */ + public void acquireAndRelease(TransactionContext transaction, ResourceName name, + LockType lockType, List releaseLocks) + throws DuplicateLockRequestException, NoLockHeldException { + // TODO(hw4_part1): implement + // You may modify any part of this method. You are not required to keep all your + // code within the given synchronized block -- in fact, + // you will have to write some code outside the synchronized block to avoid locking up + // the entire lock manager when a transaction is blocked. You are also allowed to + // move the synchronized block elsewhere if you wish. + synchronized (this) { + return; + } + } + + /** + * Acquire a LOCKTYPE lock on NAME, for transaction TRANSACTION. + * + * Error checking must be done before the lock is acquired. If the new lock + * is not compatible with another transaction's lock on the resource, or if there are + * other transaction in queue for the resource, the transaction is + * blocked and the request is placed at the **back** of NAME's queue. + * + * @throws DuplicateLockRequestException if a lock on NAME is held by + * TRANSACTION + */ + public void acquire(TransactionContext transaction, ResourceName name, + LockType lockType) throws DuplicateLockRequestException { + // TODO(hw4_part1): implement + // You may modify any part of this method. You are not required to keep all your + // code within the given synchronized block -- in fact, + // you will have to write some code outside the synchronized block to avoid locking up + // the entire lock manager when a transaction is blocked. You are also allowed to + // move the synchronized block elsewhere if you wish. + synchronized (this) { + return; + } + } + + /** + * Release TRANSACTION's lock on NAME. + * + * Error checking must be done before the lock is released. + * + * NAME's queue should be processed after this call. If any requests in + * the queue have locks to be released, those should be released, and the + * corresponding queues also processed. + * + * @throws NoLockHeldException if no lock on NAME is held by TRANSACTION + */ + public void release(TransactionContext transaction, ResourceName name) + throws NoLockHeldException { + // TODO(hw4_part1): implement + // You may modify any part of this method. + synchronized (this) { + return; + } + } + + /** + * Promote TRANSACTION's lock on NAME to NEWLOCKTYPE (i.e. change TRANSACTION's lock + * on NAME from the current lock type to NEWLOCKTYPE, which must be strictly more + * permissive). + * + * Error checking must be done before any locks are changed. If the new lock + * is not compatible with another transaction's lock on the resource, the transaction is + * blocked and the request is placed at the **front** of ITEM's queue. + * + * A lock promotion **should not** change the acquisition time of the lock, i.e. + * if a transaction acquired locks in the order: S(A), X(B), promote X(A), the + * lock on A is considered to have been acquired before the lock on B. + * + * @throws DuplicateLockRequestException if TRANSACTION already has a + * NEWLOCKTYPE lock on NAME + * @throws NoLockHeldException if TRANSACTION has no lock on NAME + * @throws InvalidLockException if the requested lock type is not a promotion. A promotion + * from lock type A to lock type B is valid if and only if B is substitutable + * for A, and B is not equal to A. + */ + public void promote(TransactionContext transaction, ResourceName name, + LockType newLockType) + throws DuplicateLockRequestException, NoLockHeldException, InvalidLockException { + // TODO(hw4_part1): implement + // You may modify any part of this method. + synchronized (this) { + return; + } + } + + /** + * Return the type of lock TRANSACTION has on NAME (return NL if no lock is held). + */ + public synchronized LockType getLockType(TransactionContext transaction, ResourceName name) { + // TODO(hw4_part1): implement + + return LockType.NL; + } + + /** + * Returns the list of locks held on NAME, in order of acquisition. + * A promotion or acquire-and-release should count as acquired + * at the original time. + */ + public synchronized List getLocks(ResourceName name) { + return new ArrayList<>(resourceEntries.getOrDefault(name, new ResourceEntry()).locks); + } + + /** + * Returns the list of locks locks held by + * TRANSACTION, in order of acquisition. A promotion or + * acquire-and-release should count as acquired at the original time. + */ + public synchronized List getLocks(TransactionContext transaction) { + return new ArrayList<>(transactionLocks.getOrDefault(transaction.getTransNum(), + Collections.emptyList())); + } + + /** + * Creates a lock context. See comments at + * he top of this file and the top of LockContext.java for more information. + */ + public synchronized LockContext context(String readable, long name) { + if (!contexts.containsKey(name)) { + contexts.put(name, new LockContext(this, null, new Pair<>(readable, name))); + } + return contexts.get(name); + } + + /** + * Create a lock context for the database. See comments at + * the top of this file and the top of LockContext.java for more information. + */ + public synchronized LockContext databaseContext() { + return context("database", 0L); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/LockRequest.java b/src/main/java/edu/berkeley/cs186/database/concurrency/LockRequest.java new file mode 100644 index 0000000..98bb87c --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/LockRequest.java @@ -0,0 +1,37 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.TransactionContext; + +import java.util.Collections; +import java.util.List; + +/** + * Represents a lock request on the queue, for + * TRANSACTION requesting LOCK and releasing everything in RELEASEDLOCKS. + * LOCK should be granted and everything in RELEASEDLOCKS should be released + * *before* the transaction is unblocked. + */ +class LockRequest { + TransactionContext transaction; + Lock lock; + List releasedLocks; + + // Lock request for LOCK, that is not releasing anything. + LockRequest(TransactionContext transaction, Lock lock) { + this.transaction = transaction; + this.lock = lock; + this.releasedLocks = Collections.emptyList(); + } + + // Lock request for LOCK, in exchange for all the locks in RELEASEDLOCKS. + LockRequest(TransactionContext transaction, Lock lock, List releasedLocks) { + this.transaction = transaction; + this.lock = lock; + this.releasedLocks = releasedLocks; + } + + @Override + public String toString() { + return "Request for " + lock.toString() + " (releasing " + releasedLocks.toString() + ")"; + } +} \ No newline at end of file diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/LockType.java b/src/main/java/edu/berkeley/cs186/database/concurrency/LockType.java new file mode 100644 index 0000000..3bdce09 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/LockType.java @@ -0,0 +1,86 @@ +package edu.berkeley.cs186.database.concurrency; + +public enum LockType { + S, // shared + X, // exclusive + IS, // intention shared + IX, // intention exclusive + SIX, // shared intention exclusive + NL; // no lock held + + /** + * This method checks whether lock types A and B are compatible with + * each other. If a transaction can hold lock type A on a resource + * at the same time another transaction holds lock type B on the same + * resource, the lock types are compatible. + */ + public static boolean compatible(LockType a, LockType b) { + if (a == null || b == null) { + throw new NullPointerException("null lock type"); + } + // TODO(hw4_part1): implement + + return false; + } + + /** + * This method returns the lock on the parent resource + * that should be requested for a lock of type A to be granted. + */ + public static LockType parentLock(LockType a) { + if (a == null) { + throw new NullPointerException("null lock type"); + } + switch (a) { + case S: return IS; + case X: return IX; + case IS: return IS; + case IX: return IX; + case SIX: return IX; + case NL: return NL; + default: throw new UnsupportedOperationException("bad lock type"); + } + } + + /** + * This method returns if parentLockType has permissions to grant a childLockType + * on a child. + */ + public static boolean canBeParentLock(LockType parentLockType, LockType childLockType) { + if (parentLockType == null || childLockType == null) { + throw new NullPointerException("null lock type"); + } + // TODO(hw4_part1): implement + + return false; + } + + /** + * This method returns whether a lock can be used for a situation + * requiring another lock (e.g. an S lock can be substituted with + * an X lock, because an X lock allows the transaction to do everything + * the S lock allowed it to do). + */ + public static boolean substitutable(LockType substitute, LockType required) { + if (required == null || substitute == null) { + throw new NullPointerException("null lock type"); + } + // TODO(hw4_part1): implement + + return false; + } + + @Override + public String toString() { + switch (this) { + case S: return "S"; + case X: return "X"; + case IS: return "IS"; + case IX: return "IX"; + case SIX: return "SIX"; + case NL: return "NL"; + default: throw new UnsupportedOperationException("bad lock type"); + } + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/LockUtil.java b/src/main/java/edu/berkeley/cs186/database/concurrency/LockUtil.java new file mode 100644 index 0000000..b59ce5c --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/LockUtil.java @@ -0,0 +1,30 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.TransactionContext; + +/** + * LockUtil is a declarative layer which simplifies multigranularity lock acquisition + * for the user (you, in the second half of Part 2). Generally speaking, you should use LockUtil + * for lock acquisition instead of calling LockContext methods directly. + */ +public class LockUtil { + /** + * Ensure that the current transaction can perform actions requiring LOCKTYPE on LOCKCONTEXT. + * + * This method should promote/escalate as needed, but should only grant the least + * permissive set of locks needed. + * + * lockType is guaranteed to be one of: S, X, NL. + * + * If the current transaction is null (i.e. there is no current transaction), this method should do nothing. + */ + public static void ensureSufficientLockHeld(LockContext lockContext, LockType lockType) { + // TODO(hw4_part2): implement + + TransactionContext transaction = TransactionContext.getTransaction(); // current transaction + + return; + } + + // TODO(hw4_part2): add helper methods as you see fit +} diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/NoLockHeldException.java b/src/main/java/edu/berkeley/cs186/database/concurrency/NoLockHeldException.java new file mode 100644 index 0000000..aef1a2e --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/NoLockHeldException.java @@ -0,0 +1,8 @@ +package edu.berkeley.cs186.database.concurrency; + +public class NoLockHeldException extends RuntimeException { + NoLockHeldException(String message) { + super(message); + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/concurrency/ResourceName.java b/src/main/java/edu/berkeley/cs186/database/concurrency/ResourceName.java new file mode 100644 index 0000000..c3da942 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/concurrency/ResourceName.java @@ -0,0 +1,95 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.common.Pair; + +import java.util.*; + +import static java.util.stream.Collectors.toList; + +/** + * This class represents the full name of a resource. The name + * of a resource is an ordered tuple of integers, and any + * subsequence of the tuple starting with the first element + * is the name of a resource higher up on the hierarchy. For debugging + * aid, we attach a string to each integer (which is only used in toString()). + * + * For example, a page may have the name (0, 3, 10), where 3 is the table's + * partition number and 10 is the page number. We store this as the list + * [("database, 0), ("Students", 3), ("10", 10)], and its ancestors on the + * hierarchy would be [("database", 0)] (which represents the entire database), + * and [("database", 0), ("Students", 3)] (which + * represents the Students table, of which this is a page of). + */ +public class ResourceName { + private final List> names; + private final int hash; + + public ResourceName(Pair name) { + this(Collections.singletonList(name)); + } + + private ResourceName(List> names) { + this.names = new ArrayList<>(names); + this.hash = names.stream().map(x -> x == null ? null : x.getSecond()).collect(toList()).hashCode(); + } + + ResourceName(ResourceName parent, Pair name) { + names = new ArrayList<>(parent.names); + names.add(name); + this.hash = names.stream().map(x -> x == null ? null : x.getSecond()).collect(toList()).hashCode(); + } + + ResourceName parent() { + if (names.size() > 1) { + return new ResourceName(names.subList(0, names.size() - 1)); + } else { + return null; + } + } + + boolean isDescendantOf(ResourceName other) { + if (other.names.size() >= names.size()) { + return false; + } + Iterator> mine = names.iterator(); + Iterator> others = other.names.iterator(); + while (others.hasNext()) { + if (!mine.next().getSecond().equals(others.next().getSecond())) { + return false; + } + } + return true; + } + + Pair getCurrentName() { + return names.get(names.size() - 1); + } + + List> getNames() { + return names; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ResourceName)) { + return false; + } + ResourceName n = (ResourceName) other; + return n.hash == hash; + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public String toString() { + StringBuilder rn = new StringBuilder(names.get(0).getFirst()); + for (int i = 1; i < names.size(); ++i) { + rn.append('/').append(names.get(i).getFirst()); + } + return rn.toString(); + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/databox/BoolDataBox.java b/src/main/java/edu/berkeley/cs186/database/databox/BoolDataBox.java new file mode 100644 index 0000000..1be779b --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/BoolDataBox.java @@ -0,0 +1,60 @@ +package edu.berkeley.cs186.database.databox; + +import java.nio.ByteBuffer; + +public class BoolDataBox extends DataBox { + private boolean b; + + public BoolDataBox(boolean b) { + this.b = b; + } + + @Override + public Type type() { + return Type.boolType(); + } + + @Override + public boolean getBool() { + return this.b; + } + + @Override + public byte[] toBytes() { + byte val = b ? (byte) 1 : (byte) 0; + return ByteBuffer.allocate(1).put(val).array(); + } + + @Override + public String toString() { + return Boolean.toString(b); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof BoolDataBox)) { + return false; + } + BoolDataBox b = (BoolDataBox) o; + return this.b == b.b; + } + + @Override + public int hashCode() { + return Boolean.valueOf(b).hashCode(); + } + + @Override + public int compareTo(DataBox d) { + if (!(d instanceof BoolDataBox)) { + String err = String.format("Invalid comparison between %s and %s.", + toString(), d.toString()); + throw new DataBoxException(err); + } + BoolDataBox b = (BoolDataBox) d; + return Boolean.compare(this.b, b.b); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/databox/DataBox.java b/src/main/java/edu/berkeley/cs186/database/databox/DataBox.java new file mode 100644 index 0000000..83e90ba --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/DataBox.java @@ -0,0 +1,95 @@ +package edu.berkeley.cs186.database.databox; + +import edu.berkeley.cs186.database.common.Buffer; + +import java.nio.charset.Charset; + +/** + * A DataBox is an element of one of the primitive types specified in + * Type.java. You can create + * + * - booleans with new BoolDataBox(b), + * - integers with new IntDataBox(i), + * - floats with new FloatDataBox(f), + * - strings with new StringDataBox(s, n), and + * - longs with new LongDataBox(l). + * + * You can unwrap a databox by first pattern matching on its type and then + * using one of getBool, getInt, getFloat, getString, and getLong: + * + * Databox d = DataBox.fromBytes(bytes); + * switch (d.type().getTypeId()) { + * case BOOL: { System.out.println(d.getBool()); } + * case INT: { System.out.println(d.getInt()); } + * case FLOAT: { System.out.println(d.getFloat()); } + * case STRING: { System.out.println(d.getString()); } + * case LONG: { System.out.println(d.getLong()); } + * } + */ +public abstract class DataBox implements Comparable { + public abstract Type type(); + + public boolean getBool() { + throw new DataBoxException("not boolean type"); + } + + public int getInt() { + throw new DataBoxException("not int type"); + } + + public float getFloat() { + throw new DataBoxException("not float type"); + } + + public String getString() { + throw new DataBoxException("not String type"); + } + + public long getLong() { + throw new DataBoxException("not Long type"); + } + + // Databoxes are serialized as follows: + // + // - BoolDataBoxes are serialized to a single byte that is 0 if the + // BoolDataBox is false and 1 if the Databox is true. + // - An IntDataBox and a FloatDataBox are serialized to their 4-byte + // values (e.g. using ByteBuffer::putInt or ByteBuffer::putFloat). + // - The first byte of a serialized m-byte StringDataBox is the 4-byte + // number m. Then come the m bytes of the string. + // + // Note that when DataBoxes are serialized, they do not serialize their type. + // That is, serialized DataBoxes are not self-descriptive; you need the type + // of a Databox in order to parse it. + public abstract byte[] toBytes(); + + public static DataBox fromBytes(Buffer buf, Type type) { + switch (type.getTypeId()) { + case BOOL: { + byte b = buf.get(); + assert (b == 0 || b == 1); + return new BoolDataBox(b == 1); + } + case INT: { + return new IntDataBox(buf.getInt()); + } + case FLOAT: { + return new FloatDataBox(buf.getFloat()); + } + case STRING: { + byte[] bytes = new byte[type.getSizeInBytes()]; + buf.get(bytes); + String s = new String(bytes, Charset.forName("UTF-8")); + return new StringDataBox(s, type.getSizeInBytes()); + } + case LONG: { + return new LongDataBox(buf.getLong()); + } + default: { + String err = String.format("Unhandled TypeId %s.", + type.getTypeId().toString()); + throw new IllegalArgumentException(err); + } + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/databox/DataBoxException.java b/src/main/java/edu/berkeley/cs186/database/databox/DataBoxException.java new file mode 100644 index 0000000..7c256ee --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/DataBoxException.java @@ -0,0 +1,7 @@ +package edu.berkeley.cs186.database.databox; + +public class DataBoxException extends RuntimeException { + public DataBoxException(String message) { + super(message); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/databox/FloatDataBox.java b/src/main/java/edu/berkeley/cs186/database/databox/FloatDataBox.java new file mode 100644 index 0000000..5fde337 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/FloatDataBox.java @@ -0,0 +1,58 @@ +package edu.berkeley.cs186.database.databox; +import java.nio.ByteBuffer; + +public class FloatDataBox extends DataBox { + private float f; + + public FloatDataBox(float f) { + this.f = f; + } + + @Override + public Type type() { + return Type.floatType(); + } + + @Override + public float getFloat() { + return this.f; + } + + @Override + public byte[] toBytes() { + return ByteBuffer.allocate(Float.BYTES).putFloat(f).array(); + } + + @Override + public String toString() { + return Float.toString(f); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FloatDataBox)) { + return false; + } + FloatDataBox f = (FloatDataBox) o; + return this.f == f.f; + } + + @Override + public int hashCode() { + return new Float(f).hashCode(); + } + + @Override + public int compareTo(DataBox d) { + if (!(d instanceof FloatDataBox)) { + String err = String.format("Invalid comparison between %s and %s.", + toString(), d.toString()); + throw new DataBoxException(err); + } + FloatDataBox f = (FloatDataBox) d; + return Float.compare(this.f, f.f); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/databox/IntDataBox.java b/src/main/java/edu/berkeley/cs186/database/databox/IntDataBox.java new file mode 100644 index 0000000..292058c --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/IntDataBox.java @@ -0,0 +1,58 @@ +package edu.berkeley.cs186.database.databox; +import java.nio.ByteBuffer; + +public class IntDataBox extends DataBox { + private int i; + + public IntDataBox(int i) { + this.i = i; + } + + @Override + public Type type() { + return Type.intType(); + } + + @Override + public int getInt() { + return this.i; + } + + @Override + public byte[] toBytes() { + return ByteBuffer.allocate(Integer.BYTES).putInt(i).array(); + } + + @Override + public String toString() { + return Integer.toString(i); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof IntDataBox)) { + return false; + } + IntDataBox i = (IntDataBox) o; + return this.i == i.i; + } + + @Override + public int hashCode() { + return new Integer(i).hashCode(); + } + + @Override + public int compareTo(DataBox d) { + if (!(d instanceof IntDataBox)) { + String err = String.format("Invalid comparison between %s and %s.", + toString(), d.toString()); + throw new DataBoxException(err); + } + IntDataBox i = (IntDataBox) d; + return Integer.compare(this.i, i.i); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/databox/LongDataBox.java b/src/main/java/edu/berkeley/cs186/database/databox/LongDataBox.java new file mode 100644 index 0000000..1378e29 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/LongDataBox.java @@ -0,0 +1,58 @@ +package edu.berkeley.cs186.database.databox; +import java.nio.ByteBuffer; + +public class LongDataBox extends DataBox { + private long l; + + public LongDataBox(long l) { + this.l = l; + } + + @Override + public Type type() { + return Type.longType(); + } + + @Override + public long getLong() { + return this.l; + } + + @Override + public byte[] toBytes() { + return ByteBuffer.allocate(Long.BYTES).putLong(l).array(); + } + + @Override + public String toString() { + return Long.toString(l); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof LongDataBox)) { + return false; + } + LongDataBox l = (LongDataBox) o; + return this.l == l.l; + } + + @Override + public int hashCode() { + return new Long(l).hashCode(); + } + + @Override + public int compareTo(DataBox d) { + if (!(d instanceof LongDataBox)) { + String err = String.format("Invalid comparison between %s and %s.", + toString(), d.toString()); + throw new DataBoxException(err); + } + LongDataBox l = (LongDataBox) d; + return Long.compare(this.l, l.l); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/databox/StringDataBox.java b/src/main/java/edu/berkeley/cs186/database/databox/StringDataBox.java new file mode 100644 index 0000000..ef7441d --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/StringDataBox.java @@ -0,0 +1,78 @@ +package edu.berkeley.cs186.database.databox; + +import java.nio.charset.Charset; + +public class StringDataBox extends DataBox { + private String s; + + // Construct an m-byte string. If s has more than m-bytes, it is truncated to + // its first m bytes. If s has fewer than m bytes, it is padded with null bytes + // until it is exactly m bytes long. + // + // - new StringDataBox("123", 5).getString() == "123\x0\x0" + // - new StringDataBox("12345", 5).getString() == "12345" + // - new StringDataBox("1234567", 5).getString() == "12345" + public StringDataBox(String s, int m) { + if (m <= 0) { + String msg = String.format("Cannot construct a %d-byte string. " + + "Strings must be at least one byte.", m); + throw new DataBoxException(msg); + } + + if (m < s.length()) { + this.s = s.substring(0, m); + } else { + this.s = s + new String(new char[m - s.length()]); + } + assert(this.s.length() == m); + } + + @Override + public Type type() { + return Type.stringType(s.length()); + } + + @Override + public String getString() { + return s.indexOf(0) < 0 ? s : s.substring(0, s.indexOf(0)); + } + + @Override + public byte[] toBytes() { + return s.getBytes(Charset.forName("ascii")); + } + + @Override + public String toString() { + // TODO(hw0): replace with return s; + return "Welcome to CS186 (original string: " + s + ")"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof StringDataBox)) { + return false; + } + StringDataBox s = (StringDataBox) o; + return this.s.equals(s.s); + } + + @Override + public int hashCode() { + return s.hashCode(); + } + + @Override + public int compareTo(DataBox d) { + if (!(d instanceof StringDataBox)) { + String err = String.format("Invalid comparison between %s and %s.", + toString(), d.toString()); + throw new DataBoxException(err); + } + StringDataBox s = (StringDataBox) d; + return this.s.compareTo(s.s); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/databox/Type.java b/src/main/java/edu/berkeley/cs186/database/databox/Type.java new file mode 100644 index 0000000..1c65f4f --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/Type.java @@ -0,0 +1,129 @@ +package edu.berkeley.cs186.database.databox; + +import edu.berkeley.cs186.database.common.Buffer; + +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * There are five primitive types: + * + * 1. 1-byte booleans (Type.boolType()), + * 2. 4-byte integers (Type.intType()), + * 3. 4-byte floats (Type.floatType()), and + * 4. n-byte strings (Type.stringType(n)) where n > 0. + * 5. 8-byte integers (Type.longType()) + * + * Note that n-byte strings and m-byte strings are considered different types + * when n != m. + */ +public class Type { + // The type of this type. + private TypeId typeId; + + // The size (in bytes) of an element of this type. + private int sizeInBytes; + + public Type(TypeId typeId, int sizeInBytes) { + this.typeId = typeId; + this.sizeInBytes = sizeInBytes; + } + + public static Type boolType() { + // Unlike all the other primitive type boxes (e.g. Integer, Float), Boolean + // does not have a BYTES field, so we hand code the fact that Java booleans + // are 1 byte. + return new Type(TypeId.BOOL, 1); + } + + public static Type intType() { + return new Type(TypeId.INT, Integer.BYTES); + } + + public static Type floatType() { + return new Type(TypeId.FLOAT, Float.BYTES); + } + + public static Type stringType(int n) { + if (n < 0) { + String msg = String.format("The provided string length %d is negative.", n); + throw new DataBoxException(msg); + } + if (n == 0) { + String msg = "Empty strings are not supported."; + throw new DataBoxException(msg); + } + return new Type(TypeId.STRING, n); + } + + public static Type longType() { + return new Type(TypeId.LONG, Long.BYTES); + } + + public TypeId getTypeId() { + return typeId; + } + + public int getSizeInBytes() { + return sizeInBytes; + } + + public byte[] toBytes() { + // A Type is uniquely identified by its typeId `t` and the size (in bytes) + // of an element of the type `s`. A Type is serialized as two integers. The + // first is the ordinal corresponding to `t`. The second is `s`. + // + // For example, the type "42-byte string" would serialized as the bytes [3, + // 42] because 3 is the ordinal of the STRING TypeId and 42 is the number + // of bytes in a 42-byte string (duh). + ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES * 2); + buf.putInt(typeId.ordinal()); + buf.putInt(sizeInBytes); + return buf.array(); + } + + public static Type fromBytes(Buffer buf) { + int ordinal = buf.getInt(); + int sizeInBytes = buf.getInt(); + switch (TypeId.fromInt(ordinal)) { + case BOOL: + assert(sizeInBytes == 1); + return Type.boolType(); + case INT: + assert(sizeInBytes == Integer.BYTES); + return Type.intType(); + case FLOAT: + assert(sizeInBytes == Float.BYTES); + return Type.floatType(); + case STRING: + return Type.stringType(sizeInBytes); + case LONG: + assert(sizeInBytes == Long.BYTES); + return Type.longType(); + default: + throw new RuntimeException("unreachable"); + } + } + + @Override + public String toString() { + return String.format("(%s, %d)", typeId.toString(), sizeInBytes); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Type)) { + return false; + } + Type t = (Type) o; + return typeId.equals(t.typeId) && sizeInBytes == t.sizeInBytes; + } + + @Override + public int hashCode() { + return Objects.hash(typeId, sizeInBytes); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/databox/TypeId.java b/src/main/java/edu/berkeley/cs186/database/databox/TypeId.java new file mode 100644 index 0000000..6331455 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/databox/TypeId.java @@ -0,0 +1,19 @@ +package edu.berkeley.cs186.database.databox; + +public enum TypeId { + BOOL, + INT, + FLOAT, + STRING, + LONG; + + private static final TypeId[] values = TypeId.values(); + + public static TypeId fromInt(int x) { + if (x < 0 || x >= values.length) { + String err = String.format("Unknown TypeId ordinal %d.", x); + throw new IllegalArgumentException(err); + } + return values[x]; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/index/BPlusNode.java b/src/main/java/edu/berkeley/cs186/database/index/BPlusNode.java new file mode 100644 index 0000000..8e9dad1 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/index/BPlusNode.java @@ -0,0 +1,270 @@ +package edu.berkeley.cs186.database.index; + +import java.util.Iterator; +import java.util.Optional; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.table.RecordId; + +/** + * An inner node or a leaf node. See InnerNode and LeafNode for more + * information. + */ +abstract class BPlusNode { + // Core API ////////////////////////////////////////////////////////////////// + /** + * n.get(k) returns the leaf node on which k may reside when queried from n. + * For example, consider the following B+ tree (for brevity, only keys are + * shown; record ids are omitted). + * + * inner + * +----+----+----+----+ + * | 10 | 20 | | | + * +----+----+----+----+ + * / | \ + * ____/ | \____ + * / | \ + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * | 1 | 2 | 3 | |->| 11 | 12 | 13 | |->| 21 | 22 | 23 | | + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * leaf0 leaf1 leaf2 + * + * inner.get(x) should return + * + * - leaf0 when x < 10, + * - leaf1 when 10 <= x < 20, and + * - leaf2 when x >= 20. + * + * Note that inner.get(4) would return leaf0 even though leaf0 doesn't + * actually contain 4. + */ + public abstract LeafNode get(DataBox key); + + /** + * n.getLeftmostLeaf() returns the leftmost leaf in the subtree rooted by n. + * In the example above, inner.getLeftmostLeaf() would return leaf0, and + * leaf1.getLeftmostLeaf() would return leaf1. + */ + public abstract LeafNode getLeftmostLeaf(); + + /** + * n.put(k, r) inserts the pair (k, r) into the subtree rooted by n. There + * are two cases to consider: + * + * Case 1: If inserting the pair (k, r) does NOT cause n to overflow, then + * Optional.empty() is returned. + * Case 2: If inserting the pair (k, r) does cause the node n to overflow, + * then n is split into a left and right node (described more + * below) and a pair (split_key, right_node_page_num) is returned + * where right_node_page_num is the page number of the newly + * created right node, and the value of split_key depends on + * whether n is an inner node or a leaf node (described more below). + * + * Now we explain how to split nodes and which split keys to return. Let's + * take a look at an example. Consider inserting the key 4 into the example + * tree above. No nodes overflow (i.e. we always hit case 1). The tree then + * looks like this: + * + * inner + * +----+----+----+----+ + * | 10 | 20 | | | + * +----+----+----+----+ + * / | \ + * ____/ | \____ + * / | \ + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * | 1 | 2 | 3 | 4 |->| 11 | 12 | 13 | |->| 21 | 22 | 23 | | + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * leaf0 leaf1 leaf2 + * + * Now let's insert key 5 into the tree. Now, leaf0 overflows and creates a + * new right sibling leaf3. d entries remain in the left node; d + 1 entries + * are moved to the right node. DO NOT REDISTRIBUTE ENTRIES ANY OTHER WAY. In + * our example, leaf0 and leaf3 would look like this: + * + * +----+----+----+----+ +----+----+----+----+ + * | 1 | 2 | | |->| 3 | 4 | 5 | | + * +----+----+----+----+ +----+----+----+----+ + * leaf0 leaf3 + * + * When a leaf splits, it returns the first entry in the right node as the + * split key. In this example, 3 is the split key. After leaf0 splits, inner + * inserts the new key and child pointer into itself and hits case 0 (i.e. it + * does not overflow). The tree looks like this: + * + * inner + * +--+--+--+--+ + * | 3|10|20| | + * +--+--+--+--+ + * / | | \ + * _______/ | | \_________ + * / | \ \ + * +--+--+--+--+ +--+--+--+--+ +--+--+--+--+ +--+--+--+--+ + * | 1| 2| | |->| 3| 4| 5| 6|->|11|12|13| |->|21|22|23| | + * +--+--+--+--+ +--+--+--+--+ +--+--+--+--+ +--+--+--+--+ + * leaf0 leaf3 leaf1 leaf2 + * + * When an inner node splits, the first d entries are kept in the left node + * and the last d entries are moved to the right node. The middle entry is + * moved (not copied) up as the split key. For example, we would split the + * following order 2 inner node + * + * +---+---+---+---+ + * | 1 | 2 | 3 | 4 | 5 + * +---+---+---+---+ + * + * into the following two inner nodes + * + * +---+---+---+---+ +---+---+---+---+ + * | 1 | 2 | | | | 4 | 5 | | | + * +---+---+---+---+ +---+---+---+---+ + * + * with a split key of 3. + * + * DO NOT redistribute entries in any other way besides what we have + * described. For example, do not move entries between nodes to avoid + * splitting. + * + * Our B+ trees do not support duplicate entries with the same key. If a + * duplicate key is inserted, the tree is left unchanged and an exception is + * raised. + */ + public abstract Optional> put(DataBox key, RecordId rid); + + /** + * n.bulkLoad(data, fillFactor) bulk loads pairs of (k, r) from data into + * the tree with the given fill factor. + * + * This method is very similar to n.put, with a couple of differences: + * + * 1. Leaf nodes do not fill up to 2*d+1 and split, but rather, fill up to + * be 1 record more than fillFactor full, then "splits" by creating a right + * sibling that contains just one record (leaving the original node with + * the desired fill factor). + * + * 2. Inner nodes should repeatedly try to bulk load the rightmost child + * until either the inner node is full (in which case it should split) + * or there is no more data. + * + * fillFactor should ONLY be used for determining how full leaf nodes are + * (not inner nodes), and calculations should round up, i.e. with d=5 + * and fillFactor=0.75, leaf nodes should be 8/10 full. + */ + public abstract Optional> bulkLoad(Iterator> data, + float fillFactor); + + /** + * n.remove(k) removes the key k and its corresponding record id from the + * subtree rooted by n, or does nothing if the key k is not in the subtree. + * REMOVE SHOULD NOT REBALANCE THE TREE. Simply delete the key and + * corresponding record id. For example, running inner.remove(2) on the + * example tree above would produce the following tree. + * + * inner + * +----+----+----+----+ + * | 10 | 20 | | | + * +----+----+----+----+ + * / | \ + * ____/ | \____ + * / | \ + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * | 1 | 3 | | |->| 11 | 12 | 13 | |->| 21 | 22 | 23 | | + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * leaf0 leaf1 leaf2 + * + * Running inner.remove(1) on this tree would produce the following tree: + * + * inner + * +----+----+----+----+ + * | 10 | 20 | | | + * +----+----+----+----+ + * / | \ + * ____/ | \____ + * / | \ + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * | 3 | | | |->| 11 | 12 | 13 | |->| 21 | 22 | 23 | | + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * leaf0 leaf1 leaf2 + * + * Running inner.remove(3) would then produce the following tree: + * + * inner + * +----+----+----+----+ + * | 10 | 20 | | | + * +----+----+----+----+ + * / | \ + * ____/ | \____ + * / | \ + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * | | | | |->| 11 | 12 | 13 | |->| 21 | 22 | 23 | | + * +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + * leaf0 leaf1 leaf2 + * + * Again, do NOT rebalance the tree. + */ + public abstract void remove(DataBox key); + + // Helpers /////////////////////////////////////////////////////////////////// + /** Get the page on which this node is persisted. */ + abstract Page getPage(); + + // Pretty Printing /////////////////////////////////////////////////////////// + /** + * S-expressions (or sexps) are a compact way of encoding nested tree-like + * structures (sort of like how JSON is a way of encoding nested dictionaries + * and lists). n.toSexp() returns an sexp encoding of the subtree rooted by + * n. For example, the following tree: + * + * +---+ + * | 3 | + * +---+ + * / \ + * +---------+---------+ +---------+---------+ + * | 1:(1 1) | 2:(2 2) | | 3:(3 3) | 4:(4 4) | + * +---------+---------+ +---------+---------+ + * + * has the following sexp + * + * (((1 (1 1)) (2 (2 2))) 3 ((3 (3 3)) (4 (4 4)))) + * + * Here, (1 (1 1)) represents the mapping from key 1 to record id (1, 1). + */ + public abstract String toSexp(); + + /** + * n.toDot() returns a fragment of a DOT file that draws the subtree rooted + * at n. + */ + public abstract String toDot(); + + // Serialization ///////////////////////////////////////////////////////////// + /** n.toBytes() serializes n. */ + public abstract byte[] toBytes(); + + /** + * BPlusNode.fromBytes(m, p) loads a BPlusNode from page `pageNum`. + */ + public static BPlusNode fromBytes(BPlusTreeMetadata metadata, BufferManager bufferManager, + LockContext treeContext, long pageNum) { + Page p = bufferManager.fetchPage(treeContext, pageNum, false); + try { + Buffer buf = p.getBuffer(); + byte b = buf.get(); + if (b == 1) { + return LeafNode.fromBytes(metadata, bufferManager, treeContext, pageNum); + } else if (b == 0) { + return InnerNode.fromBytes(metadata, bufferManager, treeContext, pageNum); + } else { + String msg = String.format("Unexpected byte %b.", b); + throw new IllegalArgumentException(msg); + } + } finally { + p.unpin(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/index/BPlusTree.java b/src/main/java/edu/berkeley/cs186/database/index/BPlusTree.java new file mode 100644 index 0000000..64d304d --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/index/BPlusTree.java @@ -0,0 +1,401 @@ +package edu.berkeley.cs186.database.index; + +import java.io.IOException; +import java.io.FileWriter; +import java.io.UncheckedIOException; +import java.util.*; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.concurrency.LockType; +import edu.berkeley.cs186.database.concurrency.LockUtil; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.table.RecordId; + +/** + * A persistent B+ tree. + * + * BPlusTree tree = new BPlusTree(bufferManager, metadata, lockContext); + * + * // Insert some values into the tree. + * tree.put(new IntDataBox(0), new RecordId(0, (short) 0)); + * tree.put(new IntDataBox(1), new RecordId(1, (short) 1)); + * tree.put(new IntDataBox(2), new RecordId(2, (short) 2)); + * + * // Get some values out of the tree. + * tree.get(new IntDataBox(0)); // Optional.of(RecordId(0, 0)) + * tree.get(new IntDataBox(1)); // Optional.of(RecordId(1, 1)) + * tree.get(new IntDataBox(2)); // Optional.of(RecordId(2, 2)) + * tree.get(new IntDataBox(3)); // Optional.empty(); + * + * // Iterate over the record ids in the tree. + * tree.scanEqual(new IntDataBox(2)); // [(2, 2)] + * tree.scanAll(); // [(0, 0), (1, 1), (2, 2)] + * tree.scanGreaterEqual(new IntDataBox(1)); // [(1, 1), (2, 2)] + * + * // Remove some elements from the tree. + * tree.get(new IntDataBox(0)); // Optional.of(RecordId(0, 0)) + * tree.remove(new IntDataBox(0)); + * tree.get(new IntDataBox(0)); // Optional.empty() + * + * // Load the tree (same as creating a new tree). + * BPlusTree fromDisk = new BPlusTree(bufferManager, metadata, lockContext); + * + * // All the values are still there. + * fromDisk.get(new IntDataBox(0)); // Optional.empty() + * fromDisk.get(new IntDataBox(1)); // Optional.of(RecordId(1, 1)) + * fromDisk.get(new IntDataBox(2)); // Optional.of(RecordId(2, 2)) + */ +public class BPlusTree { + // Buffer manager + private BufferManager bufferManager; + + // B+ tree metadata + private BPlusTreeMetadata metadata; + + // root of the B+ tree + private BPlusNode root; + + // lock context for the B+ tree + private LockContext lockContext; + + // Constructors //////////////////////////////////////////////////////////// + /** + * Construct a new B+ tree with metadata `metadata` and lock context `lockContext`. + * `metadata` contains information about the order, partition number, + * root page number, and type of keys. + * + * If the specified order is so large that a single node cannot fit on a + * single page, then a BPlusTree exception is thrown. If you want to have + * maximally full B+ tree nodes, then use the BPlusTree.maxOrder function + * to get the appropriate order. + * + * We additionally write a row to the information_schema.indices table with metadata about + * the B+ tree: + * + * - the name of the tree (table associated with it and column it indexes) + * - the key schema of the tree, + * - the order of the tree, + * - the partition number of the tree, + * - the page number of the root of the tree. + * + * All pages allocated on the given partition are serializations of inner and leaf nodes. + */ + public BPlusTree(BufferManager bufferManager, BPlusTreeMetadata metadata, LockContext lockContext) { + // TODO(hw4_part2): B+ tree locking + + // Sanity checks. + if (metadata.getOrder() < 0) { + String msg = String.format( + "You cannot construct a B+ tree with negative order %d.", + metadata.getOrder()); + throw new BPlusTreeException(msg); + } + + int maxOrder = BPlusTree.maxOrder(BufferManager.EFFECTIVE_PAGE_SIZE, metadata.getKeySchema()); + if (metadata.getOrder() > maxOrder) { + String msg = String.format( + "You cannot construct a B+ tree with order %d greater than the " + + "max order %d.", + metadata.getOrder(), maxOrder); + throw new BPlusTreeException(msg); + } + + this.bufferManager = bufferManager; + this.lockContext = lockContext; + this.metadata = metadata; + + if (this.metadata.getRootPageNum() != DiskSpaceManager.INVALID_PAGE_NUM) { + this.updateRoot(BPlusNode.fromBytes(this.metadata, bufferManager, lockContext, + this.metadata.getRootPageNum())); + } else { + // Construct the root. + List keys = new ArrayList<>(); + List rids = new ArrayList<>(); + Optional rightSibling = Optional.empty(); + this.updateRoot(new LeafNode(this.metadata, bufferManager, keys, rids, rightSibling, lockContext)); + } + } + + // Core API //////////////////////////////////////////////////////////////// + /** + * Returns the value associated with `key`. + * + * // Insert a single value into the tree. + * DataBox key = new IntDataBox(42); + * RecordId rid = new RecordId(0, (short) 0); + * tree.put(key, rid); + * + * // Get the value we put and also try to get a value we never put. + * tree.get(key); // Optional.of(rid) + * tree.get(new IntDataBox(100)); // Optional.empty() + */ + public Optional get(DataBox key) { + typecheck(key); + // TODO(hw2): implement + // TODO(hw4_part2): B+ tree locking + + return Optional.empty(); + } + + /** + * scanEqual(k) is equivalent to get(k) except that it returns an iterator + * instead of an Optional. That is, if get(k) returns Optional.empty(), + * then scanEqual(k) returns an empty iterator. If get(k) returns + * Optional.of(rid) for some rid, then scanEqual(k) returns an iterator + * over rid. + */ + public Iterator scanEqual(DataBox key) { + typecheck(key); + // TODO(hw4_part2): B+ tree locking + + Optional rid = get(key); + if (rid.isPresent()) { + ArrayList l = new ArrayList<>(); + l.add(rid.get()); + return l.iterator(); + } else { + return Collections.emptyIterator(); + } + } + + /** + * Returns an iterator over all the RecordIds stored in the B+ tree in + * ascending order of their corresponding keys. + * + * // Create a B+ tree and insert some values into it. + * BPlusTree tree = new BPlusTree("t.txt", Type.intType(), 4); + * tree.put(new IntDataBox(2), new RecordId(2, (short) 2)); + * tree.put(new IntDataBox(5), new RecordId(5, (short) 5)); + * tree.put(new IntDataBox(4), new RecordId(4, (short) 4)); + * tree.put(new IntDataBox(1), new RecordId(1, (short) 1)); + * tree.put(new IntDataBox(3), new RecordId(3, (short) 3)); + * + * Iterator iter = tree.scanAll(); + * iter.next(); // RecordId(1, 1) + * iter.next(); // RecordId(2, 2) + * iter.next(); // RecordId(3, 3) + * iter.next(); // RecordId(4, 4) + * iter.next(); // RecordId(5, 5) + * iter.next(); // NoSuchElementException + * + * Note that you CAN NOT materialize all record ids in memory and then + * return an iterator over them. Your iterator must lazily scan over the + * leaves of the B+ tree. Solutions that materialize all record ids in + * memory will receive 0 points. + */ + public Iterator scanAll() { + // TODO(hw2): Return a BPlusTreeIterator. + // TODO(hw4_part2): B+ tree locking + + return Collections.emptyIterator(); + } + + /** + * Returns an iterator over all the RecordIds stored in the B+ tree that + * are greater than or equal to `key`. RecordIds are returned in ascending + * of their corresponding keys. + * + * // Insert some values into a tree. + * tree.put(new IntDataBox(2), new RecordId(2, (short) 2)); + * tree.put(new IntDataBox(5), new RecordId(5, (short) 5)); + * tree.put(new IntDataBox(4), new RecordId(4, (short) 4)); + * tree.put(new IntDataBox(1), new RecordId(1, (short) 1)); + * tree.put(new IntDataBox(3), new RecordId(3, (short) 3)); + * + * Iterator iter = tree.scanGreaterEqual(new IntDataBox(3)); + * iter.next(); // RecordId(3, 3) + * iter.next(); // RecordId(4, 4) + * iter.next(); // RecordId(5, 5) + * iter.next(); // NoSuchElementException + * + * Note that you CAN NOT materialize all record ids in memory and then + * return an iterator over them. Your iterator must lazily scan over the + * leaves of the B+ tree. Solutions that materialize all record ids in + * memory will receive 0 points. + */ + public Iterator scanGreaterEqual(DataBox key) { + typecheck(key); + // TODO(hw2): Return a BPlusTreeIterator. + // TODO(hw4_part2): B+ tree locking + + return Collections.emptyIterator(); + } + + /** + * Inserts a (key, rid) pair into a B+ tree. If the key already exists in + * the B+ tree, then the pair is not inserted and an exception is raised. + * + * DataBox key = new IntDataBox(42); + * RecordId rid = new RecordId(42, (short) 42); + * tree.put(key, rid); // Success :) + * tree.put(key, rid); // BPlusTreeException :( + */ + public void put(DataBox key, RecordId rid) { + typecheck(key); + // TODO(hw2): implement + // TODO(hw4_part2): B+ tree locking + + return; + } + + /** + * Bulk loads data into the B+ tree. Tree should be empty and the data + * iterator should be in sorted order (by the DataBox key field) and + * contain no duplicates (no error checking is done for this). + * + * fillFactor specifies the fill factor for leaves only; inner nodes should + * be filled up to full and split in half exactly like in put. + * + * This method should raise an exception if the tree is not empty at time + * of bulk loading. If data does not meet the preconditions (contains + * duplicates or not in order), the resulting behavior is undefined. + * + * The behavior of this method should be similar to that of InnerNode's + * bulkLoad (see comments in BPlusNode.bulkLoad). + */ + public void bulkLoad(Iterator> data, float fillFactor) { + // TODO(hw2): implement + // TODO(hw4_part2): B+ tree locking + + return; + } + + /** + * Deletes a (key, rid) pair from a B+ tree. + * + * DataBox key = new IntDataBox(42); + * RecordId rid = new RecordId(42, (short) 42); + * + * tree.put(key, rid); + * tree.get(key); // Optional.of(rid) + * tree.remove(key); + * tree.get(key); // Optional.empty() + */ + public void remove(DataBox key) { + typecheck(key); + // TODO(hw2): implement + // TODO(hw4_part2): B+ tree locking + + return; + } + + // Helpers ///////////////////////////////////////////////////////////////// + /** + * Returns a sexp representation of this tree. See BPlusNode.toSexp for + * more information. + */ + public String toSexp() { + // TODO(hw4_part2): B+ tree locking + return root.toSexp(); + } + + /** + * Debugging large B+ trees is hard. To make it a bit easier, we can print + * out a B+ tree as a DOT file which we can then convert into a nice + * picture of the B+ tree. tree.toDot() returns the contents of DOT file + * which illustrates the B+ tree. The details of the file itself is not at + * all important, just know that if you call tree.toDot() and save the + * output to a file called tree.dot, then you can run this command + * + * dot -T pdf tree.dot -o tree.pdf + * + * to create a PDF of the tree. + */ + public String toDot() { + // TODO(hw4_part2): B+ tree locking + List strings = new ArrayList<>(); + strings.add("digraph g {" ); + strings.add(" node [shape=record, height=0.1];"); + strings.add(root.toDot()); + strings.add("}"); + return String.join("\n", strings); + } + + /** + * This function is very similar to toDot() except that we write + * the dot representation of the B+ tree to a dot file and then + * convert that to a PDF that will be stored in the src directory. Pass in a + * string with the ".pdf" extension included at the end (ex "tree.pdf"). + */ + public void toDotPDFFile(String filename) { + String tree_string = toDot(); + + // Writing to intermediate dot file + try { + java.io.File file = new java.io.File("tree.dot"); + FileWriter fileWriter = new FileWriter(file); + fileWriter.write(tree_string); + fileWriter.flush(); + fileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + // Running command to convert dot file to PDF + try { + Runtime.getRuntime().exec("dot -T pdf tree.dot -o " + filename); + } catch (IOException e) { + e.printStackTrace(); + throw new UncheckedIOException(e); + } + } + + /** + * Returns the largest number d such that the serialization of a LeafNode + * with 2d entries and an InnerNode with 2d keys will fit on a single page. + */ + public static int maxOrder(short pageSize, Type keySchema) { + int leafOrder = LeafNode.maxOrder(pageSize, keySchema); + int innerOrder = InnerNode.maxOrder(pageSize, keySchema); + return Math.min(leafOrder, innerOrder); + } + + /** Returns the partition number that the B+ tree resides on. */ + public int getPartNum() { + return metadata.getPartNum(); + } + + /** Save the new root page number. */ + private void updateRoot(BPlusNode newRoot) { + this.root = newRoot; + + metadata.setRootPageNum(this.root.getPage().getPageNum()); + metadata.incrementHeight(); + TransactionContext transaction = TransactionContext.getTransaction(); + if (transaction != null) { + transaction.updateIndexMetadata(metadata); + } + } + + private void typecheck(DataBox key) { + Type t = metadata.getKeySchema(); + if (!key.type().equals(t)) { + String msg = String.format("DataBox %s is not of type %s", key, t); + throw new IllegalArgumentException(msg); + } + } + + // Iterator //////////////////////////////////////////////////////////////// + private class BPlusTreeIterator implements Iterator { + // TODO(hw2): Add whatever fields and constructors you want here. + + @Override + public boolean hasNext() { + // TODO(hw2): implement + + return false; + } + + @Override + public RecordId next() { + // TODO(hw2): implement + + throw new NoSuchElementException(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/index/BPlusTreeException.java b/src/main/java/edu/berkeley/cs186/database/index/BPlusTreeException.java new file mode 100644 index 0000000..0e1bde5 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/index/BPlusTreeException.java @@ -0,0 +1,7 @@ +package edu.berkeley.cs186.database.index; + +public class BPlusTreeException extends RuntimeException { + public BPlusTreeException(String message) { + super(message); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/index/BPlusTreeMetadata.java b/src/main/java/edu/berkeley/cs186/database/index/BPlusTreeMetadata.java new file mode 100644 index 0000000..a1b35f1 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/index/BPlusTreeMetadata.java @@ -0,0 +1,88 @@ +package edu.berkeley.cs186.database.index; + +import edu.berkeley.cs186.database.databox.Type; + +/** Metadata about a B+ tree. */ +public class BPlusTreeMetadata { + // Table for which this B+ tree is for + private final String tableName; + + // Column that this B+ tree uses as a search key + private final String colName; + + // B+ trees map keys (of some type) to record ids. This is the type of the + // keys. + private final Type keySchema; + + // The order of the tree. Given a tree of order d, its inner nodes store + // between d and 2d keys and between d+1 and 2d+1 children pointers. Leaf + // nodes store between d and 2d (key, record id) pairs. Notable exceptions + // include the root node and leaf nodes that have been deleted from; these + // may contain fewer than d entries. + private final int order; + + // The partition that the B+ tree allocates pages from. Every node of the B+ tree + // is stored on a different page on this partition. + private final int partNum; + + // The page number of the root node. + private long rootPageNum; + + // The height of this tree. + private int height; + + public BPlusTreeMetadata(String tableName, String colName, Type keySchema, int order, int partNum, + long rootPageNum, int height) { + this.tableName = tableName; + this.colName = colName; + this.keySchema = keySchema; + this.order = order; + this.partNum = partNum; + this.rootPageNum = rootPageNum; + this.height = height; + } + + public BPlusTreeMetadata(String tableName, String colName) { + this(tableName, colName, Type.intType(), -1, -1, -1, -1); + } + + public String getTableName() { + return tableName; + } + + public String getColName() { + return colName; + } + + public String getName() { + return tableName + "," + colName; + } + + public Type getKeySchema() { + return keySchema; + } + + public int getOrder() { + return order; + } + + public int getPartNum() { + return partNum; + } + + public long getRootPageNum() { + return rootPageNum; + } + + void setRootPageNum(long rootPageNum) { + this.rootPageNum = rootPageNum; + } + + public int getHeight() { + return height; + } + + void incrementHeight() { + ++height; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/index/InnerNode.java b/src/main/java/edu/berkeley/cs186/database/index/InnerNode.java new file mode 100644 index 0000000..0ea4c07 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/index/InnerNode.java @@ -0,0 +1,376 @@ +package edu.berkeley.cs186.database.index; + +import java.nio.ByteBuffer; +import java.util.*; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.table.RecordId; + +/** + * A inner node of a B+ tree. Every inner node in a B+ tree of order d stores + * between d and 2d keys. An inner node with n keys stores n + 1 "pointers" to + * children nodes (where a pointer is just a page number). Moreover, every + * inner node is serialized and persisted on a single page; see toBytes and + * fromBytes for details on how an inner node is serialized. For example, here + * is an illustration of an order 2 inner node: + * + * +----+----+----+----+ + * | 10 | 20 | 30 | | + * +----+----+----+----+ + * / | | \ + */ +class InnerNode extends BPlusNode { + // Metadata about the B+ tree that this node belongs to. + private BPlusTreeMetadata metadata; + + // Buffer manager + private BufferManager bufferManager; + + // Lock context of the B+ tree + private LockContext treeContext; + + // The page on which this leaf is serialized. + private Page page; + + // The keys and child pointers of this inner node. See the comment above + // LeafNode.keys and LeafNode.rids in LeafNode.java for a warning on the + // difference between the keys and children here versus the keys and children + // stored on disk. + private List keys; + private List children; + + // Constructors ////////////////////////////////////////////////////////////// + /** + * Construct a brand new inner node. + */ + InnerNode(BPlusTreeMetadata metadata, BufferManager bufferManager, List keys, + List children, LockContext treeContext) { + this(metadata, bufferManager, bufferManager.fetchNewPage(treeContext, metadata.getPartNum(), false), + keys, children, treeContext); + } + + /** + * Construct an inner node that is persisted to page `page`. + */ + private InnerNode(BPlusTreeMetadata metadata, BufferManager bufferManager, Page page, + List keys, List children, LockContext treeContext) { + assert(keys.size() <= 2 * metadata.getOrder()); + assert(keys.size() + 1 == children.size()); + + this.metadata = metadata; + this.bufferManager = bufferManager; + this.treeContext = treeContext; + this.page = page; + this.keys = new ArrayList<>(keys); + this.children = new ArrayList<>(children); + + sync(); + page.unpin(); + } + + // Core API ////////////////////////////////////////////////////////////////// + // See BPlusNode.get. + @Override + public LeafNode get(DataBox key) { + // TODO(hw2): implement + + return null; + } + + // See BPlusNode.getLeftmostLeaf. + @Override + public LeafNode getLeftmostLeaf() { + assert(children.size() > 0); + // TODO(hw2): implement + + return null; + } + + // See BPlusNode.put. + @Override + public Optional> put(DataBox key, RecordId rid) { + // TODO(hw2): implement + + return Optional.empty(); + } + + // See BPlusNode.bulkLoad. + @Override + public Optional> bulkLoad(Iterator> data, + float fillFactor) { + // TODO(hw2): implement + + return Optional.empty(); + } + + // See BPlusNode.remove. + @Override + public void remove(DataBox key) { + // TODO(hw2): implement + + return; + } + + // Helpers /////////////////////////////////////////////////////////////////// + @Override + public Page getPage() { + return page; + } + + private BPlusNode getChild(int i) { + long pageNum = children.get(i); + return BPlusNode.fromBytes(metadata, bufferManager, treeContext, pageNum); + } + + private void sync() { + page.pin(); + try { + Buffer b = page.getBuffer(); + byte[] newBytes = toBytes(); + byte[] bytes = new byte[newBytes.length]; + b.get(bytes); + if (!Arrays.equals(bytes, newBytes)) { + page.getBuffer().put(toBytes()); + } + } finally { + page.unpin(); + } + } + + // Just for testing. + List getKeys() { + return keys; + } + + // Just for testing. + List getChildren() { + return children; + } + /** + * Returns the largest number d such that the serialization of an InnerNode + * with 2d keys will fit on a single page. + */ + static int maxOrder(short pageSize, Type keySchema) { + // A leaf node with n entries takes up the following number of bytes: + // + // 1 + 4 + (n * keySize) + ((n + 1) * 8) + // + // where + // + // - 1 is the number of bytes used to store isLeaf, + // - 4 is the number of bytes used to store n, + // - keySize is the number of bytes used to store a DataBox of type + // keySchema, and + // - 8 is the number of bytes used to store a child pointer. + // + // Solving the following equation + // + // 5 + (n * keySize) + ((n + 1) * 8) <= pageSizeInBytes + // + // we get + // + // n = (pageSizeInBytes - 13) / (keySize + 8) + // + // The order d is half of n. + int keySize = keySchema.getSizeInBytes(); + int n = (pageSize - 13) / (keySize + 8); + return n / 2; + } + + /** + * Given a list ys sorted in ascending order, numLessThanEqual(x, ys) returns + * the number of elements in ys that are less than or equal to x. For + * example, + * + * numLessThanEqual(0, Arrays.asList(1, 2, 3, 4, 5)) == 0 + * numLessThanEqual(1, Arrays.asList(1, 2, 3, 4, 5)) == 1 + * numLessThanEqual(2, Arrays.asList(1, 2, 3, 4, 5)) == 2 + * numLessThanEqual(3, Arrays.asList(1, 2, 3, 4, 5)) == 3 + * numLessThanEqual(4, Arrays.asList(1, 2, 3, 4, 5)) == 4 + * numLessThanEqual(5, Arrays.asList(1, 2, 3, 4, 5)) == 5 + * numLessThanEqual(6, Arrays.asList(1, 2, 3, 4, 5)) == 5 + * + * This helper function is useful when we're navigating down a B+ tree and + * need to decide which child to visit. For example, imagine an index node + * with the following 4 keys and 5 children pointers: + * + * +---+---+---+---+ + * | a | b | c | d | + * +---+---+---+---+ + * / | | | \ + * 0 1 2 3 4 + * + * If we're searching the tree for value c, then we need to visit child 3. + * Not coincidentally, there are also 3 values less than or equal to c (i.e. + * a, b, c). + */ + static > int numLessThanEqual(T x, List ys) { + int n = 0; + for (T y : ys) { + if (y.compareTo(x) <= 0) { + ++n; + } else { + break; + } + } + return n; + } + + static > int numLessThan(T x, List ys) { + int n = 0; + for (T y : ys) { + if (y.compareTo(x) < 0) { + ++n; + } else { + break; + } + } + return n; + } + + // Pretty Printing /////////////////////////////////////////////////////////// + @Override + public String toString() { + StringBuilder sb = new StringBuilder("("); + for (int i = 0; i < keys.size(); ++i) { + sb.append(children.get(i)).append(" ").append(keys.get(i)).append(" "); + } + sb.append(children.get(children.size() - 1)).append(")"); + return sb.toString(); + } + + @Override + public String toSexp() { + StringBuilder sb = new StringBuilder("("); + for (int i = 0; i < keys.size(); ++i) { + sb.append(getChild(i).toSexp()).append(" ").append(keys.get(i)).append(" "); + } + sb.append(getChild(children.size() - 1).toSexp()).append(")"); + return sb.toString(); + } + + /** + * An inner node on page 0 with a single key k and two children on page 1 and + * 2 is turned into the following DOT fragment: + * + * node0[label = "|k|"]; + * ... // children + * "node0":f0 -> "node1"; + * "node0":f1 -> "node2"; + */ + @Override + public String toDot() { + List ss = new ArrayList<>(); + for (int i = 0; i < keys.size(); ++i) { + ss.add(String.format("", i)); + ss.add(keys.get(i).toString()); + } + ss.add(String.format("", keys.size())); + + long pageNum = getPage().getPageNum(); + String s = String.join("|", ss); + String node = String.format(" node%d[label = \"%s\"];", pageNum, s); + + List lines = new ArrayList<>(); + lines.add(node); + for (int i = 0; i < children.size(); ++i) { + BPlusNode child = getChild(i); + long childPageNum = child.getPage().getPageNum(); + lines.add(child.toDot()); + lines.add(String.format(" \"node%d\":f%d -> \"node%d\";", + pageNum, i, childPageNum)); + } + + return String.join("\n", lines); + } + + // Serialization ///////////////////////////////////////////////////////////// + @Override + public byte[] toBytes() { + // When we serialize an inner node, we write: + // + // a. the literal value 0 (1 byte) which indicates that this node is not + // a leaf node, + // b. the number n (4 bytes) of keys this inner node contains (which is + // one fewer than the number of children pointers), + // c. the n keys, and + // d. the n+1 children pointers. + // + // For example, the following bytes: + // + // +----+-------------+----+-------------------------+-------------------------+ + // | 00 | 00 00 00 01 | 01 | 00 00 00 00 00 00 00 03 | 00 00 00 00 00 00 00 07 | + // +----+-------------+----+-------------------------+-------------------------+ + // \__/ \___________/ \__/ \_________________________________________________/ + // a b c d + // + // represent an inner node with one key (i.e. 1) and two children pointers + // (i.e. page 3 and page 7). + + // All sizes are in bytes. + int isLeafSize = 1; + int numKeysSize = Integer.BYTES; + int keysSize = metadata.getKeySchema().getSizeInBytes() * keys.size(); + int childrenSize = Long.BYTES * children.size(); + int size = isLeafSize + numKeysSize + keysSize + childrenSize; + + ByteBuffer buf = ByteBuffer.allocate(size); + buf.put((byte) 0); + buf.putInt(keys.size()); + for (DataBox key : keys) { + buf.put(key.toBytes()); + } + for (Long child : children) { + buf.putLong(child); + } + return buf.array(); + } + + /** + * Loads an inner node from page `pageNum`. + */ + public static InnerNode fromBytes(BPlusTreeMetadata metadata, + BufferManager bufferManager, LockContext treeContext, long pageNum) { + Page page = bufferManager.fetchPage(treeContext, pageNum, false); + Buffer buf = page.getBuffer(); + + assert (buf.get() == (byte) 0); + + List keys = new ArrayList<>(); + List children = new ArrayList<>(); + int n = buf.getInt(); + for (int i = 0; i < n; ++i) { + keys.add(DataBox.fromBytes(buf, metadata.getKeySchema())); + } + for (int i = 0; i < n + 1; ++i) { + children.add(buf.getLong()); + } + return new InnerNode(metadata, bufferManager, page, keys, children, treeContext); + } + + // Builtins ////////////////////////////////////////////////////////////////// + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof InnerNode)) { + return false; + } + InnerNode n = (InnerNode) o; + return page.getPageNum() == n.page.getPageNum() && + keys.equals(n.keys) && + children.equals(n.children); + } + + @Override + public int hashCode() { + return Objects.hash(page.getPageNum(), keys, children); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/index/LeafNode.java b/src/main/java/edu/berkeley/cs186/database/index/LeafNode.java new file mode 100644 index 0000000..f24ff2d --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/index/LeafNode.java @@ -0,0 +1,390 @@ +package edu.berkeley.cs186.database.index; + +import java.nio.ByteBuffer; +import java.util.*; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.table.RecordId; + +/** + * A leaf of a B+ tree. Every leaf in a B+ tree of order d stores between d and + * 2d (key, record id) pairs and a pointer to its right sibling (i.e. the page + * number of its right sibling). Moreover, every leaf node is serialized and + * persisted on a single page; see toBytes and fromBytes for details on how a + * leaf is serialized. For example, here is an illustration of two order 2 + * leafs connected together: + * + * leaf 1 (stored on some page) leaf 2 (stored on some other page) + * +-------+-------+-------+-------+ +-------+-------+-------+-------+ + * | k0:r0 | k1:r1 | k2:r2 | | --> | k3:r3 | k4:r4 | | | + * +-------+-------+-------+-------+ +-------+-------+-------+-------+ + */ +class LeafNode extends BPlusNode { + // Metadata about the B+ tree that this node belongs to. + private BPlusTreeMetadata metadata; + + // Buffer manager + private BufferManager bufferManager; + + // Lock context of the B+ tree + private LockContext treeContext; + + // The page on which this leaf is serialized. + private Page page; + + // The keys and record ids of this leaf. `keys` is always sorted in ascending + // order. The record id at index i corresponds to the key at index i. For + // example, the keys [a, b, c] and the rids [1, 2, 3] represent the pairing + // [a:1, b:2, c:3]. + // + // Note the following subtlety. keys and rids are in-memory caches of the + // keys and record ids stored on disk. Thus, consider what happens when you + // create two LeafNode objects that point to the same page: + // + // BPlusTreeMetadata meta = ...; + // int pageNum = ...; + // LockContext treeContext = new DummyLockContext(); + // + // LeafNode leaf0 = LeafNode.fromBytes(meta, bufferManager, treeContext, pageNum); + // LeafNode leaf1 = LeafNode.fromBytes(meta, bufferManager, treeContext, pageNum); + // + // This scenario looks like this: + // + // HEAP | DISK + // =============================================================== + // leaf0 | page 42 + // +-------------------------+ | +-------+-------+-------+-------+ + // | keys = [k0, k1, k2] | | | k0:r0 | k1:r1 | k2:r2 | | + // | rids = [r0, r1, r2] | | +-------+-------+-------+-------+ + // | pageNum = 42 | | + // +-------------------------+ | + // | + // leaf1 | + // +-------------------------+ | + // | keys = [k0, k1, k2] | | + // | rids = [r0, r1, r2] | | + // | pageNum = 42 | | + // +-------------------------+ | + // | + // + // Now imagine we perform on operation on leaf0 like leaf0.put(k3, r3). The + // in-memory values of leaf0 will be updated and they will be synced to disk. + // But, the in-memory values of leaf1 will not be updated. That will look + // like this: + // + // HEAP | DISK + // =============================================================== + // leaf0 | page 42 + // +-------------------------+ | +-------+-------+-------+-------+ + // | keys = [k0, k1, k2, k3] | | | k0:r0 | k1:r1 | k2:r2 | k3:r3 | + // | rids = [r0, r1, r2, r3] | | +-------+-------+-------+-------+ + // | pageNum = 42 | | + // +-------------------------+ | + // | + // leaf1 | + // +-------------------------+ | + // | keys = [k0, k1, k2] | | + // | rids = [r0, r1, r2] | | + // | pageNum = 42 | | + // +-------------------------+ | + // | + // + // Make sure your code (or your tests) doesn't use stale in-memory cached + // values of keys and rids. + private List keys; + private List rids; + + // If this leaf is the rightmost leaf, then rightSibling is Optional.empty(). + // Otherwise, rightSibling is Optional.of(n) where n is the page number of + // this leaf's right sibling. + private Optional rightSibling; + + // Constructors ////////////////////////////////////////////////////////////// + /** + * Construct a brand new leaf node. + */ + LeafNode(BPlusTreeMetadata metadata, BufferManager bufferManager, List keys, + List rids, Optional rightSibling, LockContext treeContext) { + this(metadata, bufferManager, bufferManager.fetchNewPage(treeContext, metadata.getPartNum(), false), + keys, rids, + rightSibling, treeContext); + } + + /** + * Construct a leaf node that is persisted to page `page`. + */ + private LeafNode(BPlusTreeMetadata metadata, BufferManager bufferManager, Page page, + List keys, + List rids, Optional rightSibling, LockContext treeContext) { + assert(keys.size() == rids.size()); + + this.metadata = metadata; + this.bufferManager = bufferManager; + this.treeContext = treeContext; + this.page = page; + this.keys = new ArrayList<>(keys); + this.rids = new ArrayList<>(rids); + this.rightSibling = rightSibling; + + sync(); + page.unpin(); + } + + // Core API ////////////////////////////////////////////////////////////////// + // See BPlusNode.get. + @Override + public LeafNode get(DataBox key) { + // TODO(hw2): implement + + return null; + } + + // See BPlusNode.getLeftmostLeaf. + @Override + public LeafNode getLeftmostLeaf() { + // TODO(hw2): implement + + return null; + } + + // See BPlusNode.put. + @Override + public Optional> put(DataBox key, RecordId rid) { + // TODO(hw2): implement + + return Optional.empty(); + } + + // See BPlusNode.bulkLoad. + @Override + public Optional> bulkLoad(Iterator> data, + float fillFactor) { + // TODO(hw2): implement + + return Optional.empty(); + } + + // See BPlusNode.remove. + @Override + public void remove(DataBox key) { + // TODO(hw2): implement + + return; + } + + // Iterators ///////////////////////////////////////////////////////////////// + /** Return the record id associated with `key`. */ + Optional getKey(DataBox key) { + int index = keys.indexOf(key); + return index == -1 ? Optional.empty() : Optional.of(rids.get(index)); + } + + /** + * Returns an iterator over the record ids of this leaf in ascending order of + * their corresponding keys. + */ + Iterator scanAll() { + return rids.iterator(); + } + + /** + * Returns an iterator over the record ids of this leaf that have a + * corresponding key greater than or equal to `key`. The record ids are + * returned in ascending order of their corresponding keys. + */ + Iterator scanGreaterEqual(DataBox key) { + int index = InnerNode.numLessThan(key, keys); + return rids.subList(index, rids.size()).iterator(); + } + + // Helpers /////////////////////////////////////////////////////////////////// + @Override + public Page getPage() { + return page; + } + + /** Returns the right sibling of this leaf, if it has one. */ + Optional getRightSibling() { + if (!rightSibling.isPresent()) { + return Optional.empty(); + } + + long pageNum = rightSibling.get(); + return Optional.of(LeafNode.fromBytes(metadata, bufferManager, treeContext, pageNum)); + } + + /** Serializes this leaf to its page. */ + private void sync() { + page.pin(); + try { + Buffer b = page.getBuffer(); + byte[] newBytes = toBytes(); + byte[] bytes = new byte[newBytes.length]; + b.get(bytes); + if (!Arrays.equals(bytes, newBytes)) { + page.getBuffer().put(toBytes()); + } + } finally { + page.unpin(); + } + } + + // Just for testing. + List getKeys() { + return keys; + } + + // Just for testing. + List getRids() { + return rids; + } + + /** + * Returns the largest number d such that the serialization of a LeafNode + * with 2d entries will fit on a single page. + */ + static int maxOrder(short pageSize, Type keySchema) { + // A leaf node with n entries takes up the following number of bytes: + // + // 1 + 8 + 4 + n * (keySize + ridSize) + // + // where + // + // - 1 is the number of bytes used to store isLeaf, + // - 8 is the number of bytes used to store a sibling pointer, + // - 4 is the number of bytes used to store n, + // - keySize is the number of bytes used to store a DataBox of type + // keySchema, and + // - ridSize is the number of bytes of a RecordId. + // + // Solving the following equation + // + // n * (keySize + ridSize) + 13 <= pageSizeInBytes + // + // we get + // + // n = (pageSizeInBytes - 13) / (keySize + ridSize) + // + // The order d is half of n. + int keySize = keySchema.getSizeInBytes(); + int ridSize = RecordId.getSizeInBytes(); + int n = (pageSize - 13) / (keySize + ridSize); + return n / 2; + } + + // Pretty Printing /////////////////////////////////////////////////////////// + @Override + public String toString() { + return String.format("LeafNode(pageNum=%s, keys=%s, rids=%s)", + page.getPageNum(), keys, rids); + } + + @Override + public String toSexp() { + List ss = new ArrayList<>(); + for (int i = 0; i < keys.size(); ++i) { + String key = keys.get(i).toString(); + String rid = rids.get(i).toSexp(); + ss.add(String.format("(%s %s)", key, rid)); + } + return String.format("(%s)", String.join(" ", ss)); + } + + /** + * Given a leaf with page number 1 and three (key, rid) pairs (0, (0, 0)), + * (1, (1, 1)), and (2, (2, 2)), the corresponding dot fragment is: + * + * node1[label = "{0: (0 0)|1: (1 1)|2: (2 2)}"]; + */ + @Override + public String toDot() { + List ss = new ArrayList<>(); + for (int i = 0; i < keys.size(); ++i) { + ss.add(String.format("%s: %s", keys.get(i), rids.get(i).toSexp())); + } + long pageNum = getPage().getPageNum(); + String s = String.join("|", ss); + return String.format(" node%d[label = \"{%s}\"];", pageNum, s); + } + + // Serialization ///////////////////////////////////////////////////////////// + @Override + public byte[] toBytes() { + // When we serialize a leaf node, we write: + // + // a. the literal value 1 (1 byte) which indicates that this node is a + // leaf node, + // b. the page id (8 bytes) of our right sibling (or -1 if we don't have + // a right sibling), + // c. the number (4 bytes) of (key, rid) pairs this leaf node contains, + // and + // d. the (key, rid) pairs themselves. + // + // For example, the following bytes: + // + // +----+-------------------------+-------------+----+-------------------------------+ + // | 01 | 00 00 00 00 00 00 00 04 | 00 00 00 01 | 03 | 00 00 00 00 00 00 00 03 00 01 | + // +----+-------------------------+-------------+----+-------------------------------+ + // \__/ \_______________________/ \___________/ \__________________________________/ + // a b c d + // + // represent a leaf node with sibling on page 4 and a single (key, rid) + // pair with key 3 and page id (3, 1). + + // All sizes are in bytes. + int isLeafSize = 1; + int siblingSize = Long.BYTES; + int lenSize = Integer.BYTES; + int keySize = metadata.getKeySchema().getSizeInBytes(); + int ridSize = RecordId.getSizeInBytes(); + int entriesSize = (keySize + ridSize) * keys.size(); + int size = isLeafSize + siblingSize + lenSize + entriesSize; + + ByteBuffer buf = ByteBuffer.allocate(size); + buf.put((byte) 1); + buf.putLong(rightSibling.orElse(-1L)); + buf.putInt(keys.size()); + for (int i = 0; i < keys.size(); ++i) { + buf.put(keys.get(i).toBytes()); + buf.put(rids.get(i).toBytes()); + } + return buf.array(); + } + + /** + * Loads a leaf node from page `pageNum`. + */ + public static LeafNode fromBytes(BPlusTreeMetadata metadata, BufferManager bufferManager, + LockContext treeContext, long pageNum) { + // TODO(hw2): implement + + return null; + } + + // Builtins ////////////////////////////////////////////////////////////////// + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof LeafNode)) { + return false; + } + LeafNode n = (LeafNode) o; + return page.getPageNum() == n.page.getPageNum() && + keys.equals(n.keys) && + rids.equals(n.rids) && + rightSibling.equals(n.rightSibling); + } + + @Override + public int hashCode() { + return Objects.hash(page.getPageNum(), keys, rids, rightSibling); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/io/DiskSpaceManager.java b/src/main/java/edu/berkeley/cs186/database/io/DiskSpaceManager.java new file mode 100644 index 0000000..ba17dbd --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/io/DiskSpaceManager.java @@ -0,0 +1,104 @@ +package edu.berkeley.cs186.database.io; + +public interface DiskSpaceManager extends AutoCloseable { + short PAGE_SIZE = 4096; // size of a page in bytes + long INVALID_PAGE_NUM = -1L; // a page number that is always invalid + + @Override + void close(); + + /** + * Allocates a new partition. + * + * @return partition number of new partition + */ + int allocPart(); + + /** + * Allocates a new partition with a specific partition number. + * + * @param partNum partition number of new partition + * @return partition number of new partition + */ + int allocPart(int partNum); + + /** + * Releases a partition from use. + + * @param partNum partition number to be released + */ + void freePart(int partNum); + + /** + * Allocates a new page. + * @param partNum partition to allocate new page under + * @return virtual page number of new page + */ + long allocPage(int partNum); + + /** + * Allocates a new page with a specific page number. + * @param pageNum page number of new page + * @return virtual page number of new page + */ + long allocPage(long pageNum); + + /** + * Frees a page. The page cannot be used after this call. + * @param page virtual page number of page to be released + */ + void freePage(long page); + + /** + * Reads a page. + * + * @param page number of page to be read + * @param buf byte buffer whose contents will be filled with page data + */ + void readPage(long page, byte[] buf); + + /** + * Writes to a page. + * + * @param page number of page to be read + * @param buf byte buffer that contains the new page data + */ + void writePage(long page, byte[] buf); + + /** + * Checks if a page is allocated + * + * @param page number of page to check + * @return true if the page is allocated, false otherwise + */ + boolean pageAllocated(long page); + + /** + * Gets partition number from virtual page number + * @param page virtual page number + * @return partition number + */ + static int getPartNum(long page) { + return (int) (page / 10000000000L); + } + + /** + * Gets data page number from virtual page number + * @param page virtual page number + * @return data page number + */ + static int getPageNum(long page) { + return (int) (page % 10000000000L); + } + + /** + * Gets the virtual page number given partition/data page number + * @param partNum partition number + * @param pageNum data page number + * @return virtual page number + */ + static long getVirtualPageNum(int partNum, int pageNum) { + return partNum * 10000000000L + pageNum; + } + +} diff --git a/src/main/java/edu/berkeley/cs186/database/io/DiskSpaceManagerImpl.java b/src/main/java/edu/berkeley/cs186/database/io/DiskSpaceManagerImpl.java new file mode 100644 index 0000000..f6371ed --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/io/DiskSpaceManagerImpl.java @@ -0,0 +1,631 @@ +package edu.berkeley.cs186.database.io; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Bits; +import edu.berkeley.cs186.database.recovery.RecoveryManager; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +/** + * An implementation of a disk space manager with virtual page translation, and + * two levels of header pages, allowing for (with page size of 4K) 256G worth of data per partition: + * + * [master page] + * / | \ + * [header page] [header page] + * / | | | \ ... / | | | \ + * [data] [data] ... [data] [data] [data] [data] ... [data] [data] + * + * Each header page stores a bitmap, indicating whether each of the data pages has been allocated, + * and manages 32K pages. The master page stores 16-bit integers for each of the header pages indicating + * the number of data pages that have been allocated under the header page (managing 2K header pages). + * A single partition may therefore have a maximum of 64M data pages. + * + * Master and header pages are cached permanently in memory; changes to these are immediately flushed to + * disk. This imposes a fairly small memory overhead (128M partitions have 2 pages cached). This caching + * is done separately from the buffer manager's caching. + * + * Virtual page numbers are 64-bit integers (Java longs) assigned to data pages in the following format: + * partition number * 10^10 + n + * for the n-th data page of the partition (indexed from 0). This particular format (instead of a simpler + * scheme such as assigning the upper 32 bits to partition number and lower 32 to page number) was chosen + * for ease of debugging (it's easier to read 10000000006 as part 1 page 6, than it is to decipher 4294967302). + * + * Partitions are backed by OS level files (one OS level file per partition), and are stored in the following + * manner: + * - the master page is the 0th page of the OS file + * - the first header page is the 1st page of the OS file + * - the next 32K pages are data pages managed by the first header page + * - the second header page follows + * - the next 32K pages are data pages managed by the second header page + * - etc. + */ +public class DiskSpaceManagerImpl implements DiskSpaceManager { + private static final int MAX_HEADER_PAGES = PAGE_SIZE / 2; // 2 bytes per header page + private static final int DATA_PAGES_PER_HEADER = PAGE_SIZE * 8; // 1 bit per data page + + // Name of base directory. + private String dbDir; + + // Info about each partition. + private Map partInfo; + + // Counter to generate new partition numbers. + private AtomicInteger partNumCounter; + + // Lock on the entire manager. + private ReentrantLock managerLock; + + // recovery manager + private RecoveryManager recoveryManager; + + private static class PartInfo implements AutoCloseable { + // Underyling OS file/file channel. + private RandomAccessFile file; + private FileChannel fileChannel; + + // Lock on the partition. + private ReentrantLock partitionLock; + + // Contents of the master page of this partition + private int[] masterPage; + + // Contents of the various header pages of this partition + private List headerPages; + + // Recovery manager + private RecoveryManager recoveryManager; + + // Partition number + private int partNum; + + private PartInfo(int partNum, RecoveryManager recoveryManager) { + this.masterPage = new int[MAX_HEADER_PAGES]; + this.headerPages = new ArrayList<>(); + this.partitionLock = new ReentrantLock(); + this.recoveryManager = recoveryManager; + this.partNum = partNum; + } + + /** + * Opens the OS file and loads master and header pages. + * @param fileName name of OS file partition is stored in + */ + private void open(String fileName) { + assert (this.fileChannel == null); + try { + this.file = new RandomAccessFile(fileName, "rw"); + this.fileChannel = this.file.getChannel(); + long length = this.file.length(); + if (length == 0) { + // new file, write empty master page and fill headerPages with null + for (int i = 0; i < MAX_HEADER_PAGES; ++i) { + this.headerPages.add(null); + } + this.writeMasterPage(); + } else { + // old file, read in master page + header pages + ByteBuffer b = ByteBuffer.wrap(new byte[PAGE_SIZE]); + this.fileChannel.read(b, PartInfo.masterPageOffset()); + b.position(0); + for (int i = 0; i < MAX_HEADER_PAGES; ++i) { + this.masterPage[i] = (b.getShort() & 0xFFFF); + if (PartInfo.headerPageOffset(i) >= length) { + this.headerPages.add(null); + } else { + byte[] headerPage = new byte[PAGE_SIZE]; + this.headerPages.add(headerPage); + this.fileChannel.read(ByteBuffer.wrap(headerPage), PartInfo.headerPageOffset(i)); + } + } + } + } catch (IOException e) { + throw new PageException("Could not open or read file: " + e.getMessage()); + } + } + + @Override + public void close() throws IOException { + this.partitionLock.lock(); + try { + this.headerPages.clear(); + this.file.close(); + this.fileChannel.close(); + } finally { + this.partitionLock.unlock(); + } + } + + /** + * Writes the master page to disk. + */ + private void writeMasterPage() throws IOException { + ByteBuffer b = ByteBuffer.wrap(new byte[PAGE_SIZE]); + for (int i = 0; i < MAX_HEADER_PAGES; ++i) { + b.putShort((short) (masterPage[i] & 0xFFFF)); + } + b.position(0); + this.fileChannel.write(b, PartInfo.masterPageOffset()); + } + + /** + * Writes a header page to disk. + * @param headerIndex which header page + */ + private void writeHeaderPage(int headerIndex) throws IOException { + ByteBuffer b = ByteBuffer.wrap(this.headerPages.get(headerIndex)); + this.fileChannel.write(b, PartInfo.headerPageOffset(headerIndex)); + } + + /** + * Allocates a new page in the partition. + * @return data page number + */ + private int allocPage() throws IOException { + int headerIndex = -1; + for (int i = 0; i < MAX_HEADER_PAGES; ++i) { + if (this.masterPage[i] < DATA_PAGES_PER_HEADER) { + headerIndex = i; + break; + } + } + if (headerIndex == -1) { + throw new PageException("no free pages - partition has reached max size"); + } + + byte[] headerBytes = this.headerPages.get(headerIndex); + + int pageIndex = -1; + if (headerBytes == null) { + pageIndex = 0; + } else { + for (int i = 0; i < DATA_PAGES_PER_HEADER; i++) { + if (Bits.getBit(headerBytes, i) == Bits.Bit.ZERO) { + pageIndex = i; + break; + } + } + if (pageIndex == -1) { + throw new PageException("header page should have free space, but doesn't"); + } + } + + return this.allocPage(headerIndex, pageIndex); + } + + /** + * Allocates a new page in the partition. + * @param headerIndex index of header page managing new page + * @param pageIndex index within header page of new page + * @return data page number + */ + private int allocPage(int headerIndex, int pageIndex) throws IOException { + byte[] headerBytes = this.headerPages.get(headerIndex); + if (headerBytes == null) { + headerBytes = new byte[PAGE_SIZE]; + this.headerPages.remove(headerIndex); + this.headerPages.add(headerIndex, headerBytes); + } + + if (Bits.getBit(headerBytes, pageIndex) == Bits.Bit.ONE) { + throw new IllegalStateException("page at (part=" + partNum + ", header=" + headerIndex + ", index=" + + + pageIndex + ") already allocated"); + } + + Bits.setBit(headerBytes, pageIndex, Bits.Bit.ONE); + this.masterPage[headerIndex] = Bits.countBits(headerBytes); + + int pageNum = pageIndex + headerIndex * DATA_PAGES_PER_HEADER; + + TransactionContext transaction = TransactionContext.getTransaction(); + if (transaction != null) { + long vpn = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + recoveryManager.logAllocPage(transaction.getTransNum(), vpn); + recoveryManager.diskIOHook(vpn); + } + + this.writeMasterPage(); + this.writeHeaderPage(headerIndex); + + return pageNum; + } + + /** + * Frees a page in the partition from use. + * @param pageNum data page number to be freed + */ + private void freePage(int pageNum) throws IOException { + int headerIndex = pageNum / DATA_PAGES_PER_HEADER; + int pageIndex = pageNum % DATA_PAGES_PER_HEADER; + + byte[] headerBytes = headerPages.get(headerIndex); + if (headerBytes == null) { + throw new NoSuchElementException("cannot free unallocated page"); + } + + if (Bits.getBit(headerBytes, pageIndex) == Bits.Bit.ZERO) { + throw new NoSuchElementException("cannot free unallocated page"); + } + + Bits.setBit(headerBytes, pageIndex, Bits.Bit.ONE); + this.masterPage[headerIndex] = Bits.countBits(headerBytes); + + TransactionContext transaction = TransactionContext.getTransaction(); + if (transaction != null) { + long vpn = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + recoveryManager.logFreePage(transaction.getTransNum(), vpn); + recoveryManager.diskIOHook(vpn); + } + + this.writeMasterPage(); + this.writeHeaderPage(headerIndex); + } + + /** + * Reads in a data page. Assumes that the partition lock is held. + * @param pageNum data page number to read in + * @param buf output buffer to be filled with page - assumed to be page size + */ + private void readPage(int pageNum, byte[] buf) throws IOException { + if (this.isNotAllocatedPage(pageNum)) { + throw new PageException("page " + pageNum + " is not allocated"); + } + ByteBuffer b = ByteBuffer.wrap(buf); + this.fileChannel.read(b, PartInfo.dataPageOffset(pageNum)); + } + + /** + * Writes to a data page. Assumes that the partition lock is held. + * @param pageNum data page number to write to + * @param buf input buffer with new contents of page - assumed to be page size + */ + private void writePage(int pageNum, byte[] buf) throws IOException { + if (this.isNotAllocatedPage(pageNum)) { + throw new PageException("page " + pageNum + " is not allocated"); + } + ByteBuffer b = ByteBuffer.wrap(buf); + this.fileChannel.write(b, PartInfo.dataPageOffset(pageNum)); + this.fileChannel.force(false); + + long vpn = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + recoveryManager.diskIOHook(vpn); + } + + /** + * Checks if page number is for an unallocated data page + * @param pageNum data page number + * @return true if page is not valid or not allocated + */ + private boolean isNotAllocatedPage(int pageNum) { + int headerIndex = pageNum / DATA_PAGES_PER_HEADER; + int pageIndex = pageNum % DATA_PAGES_PER_HEADER; + if (headerIndex < 0 || headerIndex >= MAX_HEADER_PAGES) { + return true; + } + if (masterPage[headerIndex] == 0) { + return true; + } + return Bits.getBit(headerPages.get(headerIndex), pageIndex) == Bits.Bit.ZERO; + } + + /** + * @return offset in OS file for master page + */ + private static long masterPageOffset() { + return 0; + } + + /** + * @param headerIndex which header page + * @return offset in OS file for header page + */ + private static long headerPageOffset(int headerIndex) { + return (long) (1 + headerIndex * DATA_PAGES_PER_HEADER) * PAGE_SIZE; + } + + /** + * @param pageNum data page number + * @return offset in OS file for data page + */ + private static long dataPageOffset(int pageNum) { + return (long) (2 + pageNum / DATA_PAGES_PER_HEADER + pageNum) * PAGE_SIZE; + } + + private void freeDataPages() throws IOException { + for (int i = 0; i < MAX_HEADER_PAGES; ++i) { + if (masterPage[i] > 0) { + byte[] headerPage = headerPages.get(i); + for (int j = 0; j < DATA_PAGES_PER_HEADER; ++j) { + if (Bits.getBit(headerPage, j) == Bits.Bit.ONE) { + this.freePage(i * DATA_PAGES_PER_HEADER + j); + } + } + } + } + } + } + + /** + * Initialize the disk space manager using the given directory. Creates the directory + * if not present. + * + * @param dbDir base directory of the database + */ + public DiskSpaceManagerImpl(String dbDir, RecoveryManager recoveryManager) { + this.dbDir = dbDir; + this.recoveryManager = recoveryManager; + this.partInfo = new HashMap<>(); + this.partNumCounter = new AtomicInteger(0); + this.managerLock = new ReentrantLock(); + + File dir = new File(dbDir); + if (!dir.exists()) { + if (!dir.mkdirs()) { + throw new PageException("could not initialize disk space manager - could not make directory"); + } + } else { + int maxFileNum = -1; + File[] files = dir.listFiles(); + if (files == null) { + throw new PageException("could not initialize disk space manager - directory is a file"); + } + for (File f : files) { + if (f.length() == 0) { + if (!f.delete()) { + throw new PageException("could not clean up unused file - " + f.getName()); + } + continue; + } + int fileNum = Integer.parseInt(f.getName()); + maxFileNum = Math.max(maxFileNum, fileNum); + + PartInfo pi = new PartInfo(fileNum, recoveryManager); + pi.open(dbDir + "/" + f.getName()); + this.partInfo.put(fileNum, pi); + } + this.partNumCounter.set(maxFileNum + 1); + } + } + + @Override + public void close() { + for (Map.Entry part : this.partInfo.entrySet()) { + try { + part.getValue().close(); + } catch (IOException e) { + throw new PageException("could not close partition " + part.getKey() + ": " + e.getMessage()); + } + } + } + + @Override + public int allocPart() { + return this.allocPartHelper(this.partNumCounter.getAndIncrement()); + } + + @Override + public int allocPart(int partNum) { + this.partNumCounter.updateAndGet((int x) -> Math.max(x, partNum) + 1); + return this.allocPartHelper(partNum); + } + + private int allocPartHelper(int partNum) { + PartInfo pi; + + this.managerLock.lock(); + try { + if (this.partInfo.containsKey(partNum)) { + throw new IllegalStateException("partition number " + partNum + " already exists"); + } + + pi = new PartInfo(partNum, recoveryManager); + this.partInfo.put(partNum, pi); + + pi.partitionLock.lock(); + } finally { + this.managerLock.unlock(); + } + try { + // We must open partition only after logging, but we need to release the + // manager lock first, in case the log manager is currently in the process + // of allocating a new log page (for another txn's records). + TransactionContext transaction = TransactionContext.getTransaction(); + if (transaction != null) { + recoveryManager.logAllocPart(transaction.getTransNum(), partNum); + } + + pi.open(dbDir + "/" + partNum); + return partNum; + } finally { + pi.partitionLock.unlock(); + } + } + + @Override + public void freePart(int partNum) { + PartInfo pi; + + this.managerLock.lock(); + try { + pi = this.partInfo.remove(partNum); + if (pi == null) { + throw new NoSuchElementException("no partition " + partNum); + } + pi.partitionLock.lock(); + } finally { + this.managerLock.unlock(); + } + try { + try { + pi.freeDataPages(); + pi.close(); + } catch (IOException e) { + throw new PageException("could not close partition " + partNum + ": " + e.getMessage()); + } + + TransactionContext transaction = TransactionContext.getTransaction(); + if (transaction != null) { + recoveryManager.logFreePart(transaction.getTransNum(), partNum); + } + + File pf = new File(dbDir + "/" + partNum); + if (!pf.delete()) { + throw new PageException("could not delete files for partition " + partNum); + } + } finally { + pi.partitionLock.unlock(); + } + } + + @Override + public long allocPage(int partNum) { + this.managerLock.lock(); + PartInfo pi; + try { + pi = getPartInfo(partNum); + pi.partitionLock.lock(); + } finally { + this.managerLock.unlock(); + } + try { + int pageNum = pi.allocPage(); + pi.writePage(pageNum, new byte[PAGE_SIZE]); + return DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + } catch (IOException e) { + throw new PageException("could not modify partition " + partNum + ": " + e.getMessage()); + } finally { + pi.partitionLock.unlock(); + } + } + + @Override + public long allocPage(long page) { + int partNum = DiskSpaceManager.getPartNum(page); + int pageNum = DiskSpaceManager.getPageNum(page); + int headerIndex = pageNum / DATA_PAGES_PER_HEADER; + int pageIndex = pageNum % DATA_PAGES_PER_HEADER; + + this.managerLock.lock(); + PartInfo pi; + try { + pi = getPartInfo(partNum); + pi.partitionLock.lock(); + } finally { + this.managerLock.unlock(); + } + try { + pi.allocPage(headerIndex, pageIndex); + pi.writePage(pageNum, new byte[PAGE_SIZE]); + return DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + } catch (IOException e) { + throw new PageException("could not modify partition " + partNum + ": " + e.getMessage()); + } finally { + pi.partitionLock.unlock(); + } + } + + @Override + public void freePage(long page) { + int partNum = DiskSpaceManager.getPartNum(page); + int pageNum = DiskSpaceManager.getPageNum(page); + this.managerLock.lock(); + PartInfo pi; + try { + pi = getPartInfo(partNum); + pi.partitionLock.lock(); + } finally { + this.managerLock.unlock(); + } + try { + pi.freePage(pageNum); + } catch (IOException e) { + throw new PageException("could not modify partition " + partNum + ": " + e.getMessage()); + } finally { + pi.partitionLock.unlock(); + } + } + + @Override + public void readPage(long page, byte[] buf) { + if (buf.length != PAGE_SIZE) { + throw new IllegalArgumentException("readPage expects a page-sized buffer"); + } + int partNum = DiskSpaceManager.getPartNum(page); + int pageNum = DiskSpaceManager.getPageNum(page); + this.managerLock.lock(); + PartInfo pi; + try { + pi = getPartInfo(partNum); + pi.partitionLock.lock(); + } finally { + this.managerLock.unlock(); + } + try { + pi.readPage(pageNum, buf); + } catch (IOException e) { + throw new PageException("could not read partition " + partNum + ": " + e.getMessage()); + } finally { + pi.partitionLock.unlock(); + } + } + + @Override + public void writePage(long page, byte[] buf) { + if (buf.length != PAGE_SIZE) { + throw new IllegalArgumentException("writePage expects a page-sized buffer"); + } + int partNum = DiskSpaceManager.getPartNum(page); + int pageNum = DiskSpaceManager.getPageNum(page); + this.managerLock.lock(); + PartInfo pi; + try { + pi = getPartInfo(partNum); + pi.partitionLock.lock(); + } finally { + this.managerLock.unlock(); + } + try { + pi.writePage(pageNum, buf); + } catch (IOException e) { + throw new PageException("could not write partition " + partNum + ": " + e.getMessage()); + } finally { + pi.partitionLock.unlock(); + } + } + + @Override + public boolean pageAllocated(long page) { + int partNum = DiskSpaceManager.getPartNum(page); + int pageNum = DiskSpaceManager.getPageNum(page); + this.managerLock.lock(); + PartInfo pi; + try { + pi = getPartInfo(partNum); + pi.partitionLock.lock(); + } finally { + this.managerLock.unlock(); + } + try { + return !pi.isNotAllocatedPage(pageNum); + } finally { + pi.partitionLock.unlock(); + } + } + + // Gets PartInfo, throws exception if not found. + private PartInfo getPartInfo(int partNum) { + PartInfo pi = this.partInfo.get(partNum); + if (pi == null) { + throw new NoSuchElementException("no partition " + partNum); + } + return pi; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/io/PageException.java b/src/main/java/edu/berkeley/cs186/database/io/PageException.java new file mode 100644 index 0000000..2e5cbd0 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/io/PageException.java @@ -0,0 +1,19 @@ +package edu.berkeley.cs186.database.io; + +/** + * Exception thrown for errors while paging. + */ +public class PageException extends RuntimeException { + public PageException() { + super(); + } + + public PageException(Exception e) { + super(e); + } + + public PageException(String message) { + super(message); + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/memory/BufferFrame.java b/src/main/java/edu/berkeley/cs186/database/memory/BufferFrame.java new file mode 100644 index 0000000..d362426 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/memory/BufferFrame.java @@ -0,0 +1,88 @@ +package edu.berkeley.cs186.database.memory; + +/** + * Buffer frame. + */ +abstract class BufferFrame { + Object tag = null; + private int pinCount = 0; + + /** + * Pin buffer frame; cannot be evicted while pinned. A "hit" happens when the + * buffer frame gets pinned. + */ + void pin() { + ++pinCount; + } + + /** + * Unpin buffer frame. + */ + void unpin() { + if (!isPinned()) { + throw new IllegalStateException("cannot unpin unpinned frame"); + } + --pinCount; + } + + /** + * @return whether this frame is pinned + */ + boolean isPinned() { + return pinCount > 0; + } + + /** + * @return whether this frame is valid + */ + abstract boolean isValid(); + + /** + * @return page number of this frame + */ + abstract long getPageNum(); + + /** + * Flushes this buffer frame to disk, but does not unload it. + */ + abstract void flush(); + + /** + * Read from the buffer frame. + * @param position position in buffer frame to start reading + * @param num number of bytes to read + * @param buf output buffer + */ + abstract void readBytes(short position, short num, byte[] buf); + + /** + * Write to the buffer frame, and mark frame as dirtied. + * @param position position in buffer frame to start writing + * @param num number of bytes to write + * @param buf input buffer + */ + abstract void writeBytes(short position, short num, byte[] buf); + + /** + * Requests a valid Frame object for the page (if invalid, a new Frame object is returned). + * Frame is pinned on return. + */ + abstract BufferFrame requestValidFrame(); + + /** + * @return amount of space available to user of the frame + */ + short getEffectivePageSize() { + return BufferManager.EFFECTIVE_PAGE_SIZE; + } + + /** + * @param pageLSN new pageLSN of the page loaded in this frame + */ + abstract void setPageLSN(long pageLSN); + + /** + * @return pageLSN of the page loaded in this frame + */ + abstract long getPageLSN(); +} diff --git a/src/main/java/edu/berkeley/cs186/database/memory/BufferManager.java b/src/main/java/edu/berkeley/cs186/database/memory/BufferManager.java new file mode 100644 index 0000000..4ec7e34 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/memory/BufferManager.java @@ -0,0 +1,104 @@ +package edu.berkeley.cs186.database.memory; + +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; + +import java.util.function.BiConsumer; + +public interface BufferManager extends AutoCloseable { + // We reserve 36 bytes on each page for bookkeeping for recovery + // (used to store the pageLSN, and to ensure that a redo-only/undo-only log record can + // fit on one page). + short RESERVED_SPACE = 36; + + // Effective page size available to users of buffer manager. + short EFFECTIVE_PAGE_SIZE = DiskSpaceManager.PAGE_SIZE - RESERVED_SPACE; + + @Override + void close(); + + /** + * Fetches the specified page, with a loaded and pinned buffer frame. + * + * @param parentContext lock context of the **parent** of the page being fetched + * @param pageNum page number + * @param logPage whether the page is for the log or not + * @return specified page + */ + Page fetchPage(LockContext parentContext, long pageNum, boolean logPage); + + /** + * Fetches a new page, with a loaded and pinned buffer frame. + * + * @param parentContext parent lock context of the new page + * @param partNum partition number for new page + * @param logPage whether the page is for the log or not + * @return the new page + */ + Page fetchNewPage(LockContext parentContext, int partNum, boolean logPage); + + /** + * Frees a page - evicts the page from cache, and tells the disk space manager + * that the page is no longer needed. Page must be pinned before this call, + * and cannot be used after this call (aside from unpinning). + * + * @param page page to free + */ + void freePage(Page page); + + /** + * Frees a partition - evicts all relevant pages from cache, and tells the disk space manager + * that the partition is no longer needed. No pages in the partition may be pinned before this call, + * and cannot be used after this call. + * + * @param partNum partition number to free + */ + void freePart(int partNum); + + /** + * Fetches a buffer frame with data for the specified page. Reuses existing buffer frame + * if page already loaded in memory. Pins the buffer frame. Cannot be used outside the package. + * + * @param pageNum page number + * @param logPage whether the page is for the log or not + * @return buffer frame with specified page loaded + */ + BufferFrame fetchPageFrame(long pageNum, boolean logPage); + + /** + * Fetches a buffer frame for a new page. Pins the buffer frame. Cannot be used outside the package. + * + * @param partNum partition number for new page + * @param logPage whether the page is for the log or not + * @return buffer frame for the new page + */ + BufferFrame fetchNewPageFrame(int partNum, boolean logPage); + + /** + * Calls flush on the frame of a page and unloads the page from the frame. If the page + * is not loaded, this does nothing. + * @param pageNum page number of page to evict + */ + void evict(long pageNum); + + /** + * Calls evict on every frame in sequence. + */ + void evictAll(); + + /** + * Calls the passed in method with the page number of every loaded page. + * @param process method to consume page numbers. The first parameter is the page number, + * and the second parameter is a boolean indicating whether the page is dirty + * (has an unflushed change). + */ + void iterPageNums(BiConsumer process); + + /** + * Get the number of I/Os since the buffer manager was started, excluding anything used in disk + * space management, and not counting allocation/free. This is not really useful except as a + * relative measure. + * @return number of I/Os + */ + long getNumIOs(); +} diff --git a/src/main/java/edu/berkeley/cs186/database/memory/BufferManagerImpl.java b/src/main/java/edu/berkeley/cs186/database/memory/BufferManagerImpl.java new file mode 100644 index 0000000..f21b61a --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/memory/BufferManagerImpl.java @@ -0,0 +1,561 @@ +package edu.berkeley.cs186.database.memory; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.PageException; +import edu.berkeley.cs186.database.recovery.RecoveryManager; + +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiConsumer; + +/** + * Implementation of a buffer manager, with configurable page replacement policies. + * Data is stored in page-sized byte arrays, and returned in a Frame object specific + * to the page loaded (evicting and loading a new page into the frame will result in + * a new Frame object, with the same underlying byte array), with old Frame objects + * backed by the same byte array marked as invalid. + */ +public class BufferManagerImpl implements BufferManager { + // Buffer frames + private Frame[] frames; + + // Reference to the disk space manager underneath this buffer manager instance. + private DiskSpaceManager diskSpaceManager; + + // Map of page number to frame index + private Map pageToFrame; + + // Lock on buffer manager + private ReentrantLock managerLock; + + // Eviction policy + private EvictionPolicy evictionPolicy; + + // Index of first free frame + private int firstFreeIndex; + + // Recovery manager + private RecoveryManager recoveryManager; + + // Count of number of I/Os + private long numIOs = 0; + + /** + * Buffer frame, containing information about the loaded page, wrapped around the + * underlying byte array. Free frames use the index field to create a (singly) linked + * list between free frames. + */ + class Frame extends BufferFrame { + private static final int INVALID_INDEX = Integer.MIN_VALUE; + + byte[] contents; + private int index; + private long pageNum; + private boolean dirty; + private ReentrantLock frameLock; + private boolean logPage; + + Frame(byte[] contents, int nextFree, boolean logPage) { + this(contents, ~nextFree, DiskSpaceManager.INVALID_PAGE_NUM, logPage); + } + + Frame(Frame frame) { + this(frame.contents, frame.index, frame.pageNum, frame.logPage); + } + + Frame(byte[] contents, int index, long pageNum, boolean logPage) { + this.contents = contents; + this.index = index; + this.pageNum = pageNum; + this.dirty = false; + this.frameLock = new ReentrantLock(); + this.logPage = logPage; + } + + /** + * Pin buffer frame; cannot be evicted while pinned. A "hit" happens when the + * buffer frame gets pinned. + */ + @Override + public void pin() { + this.frameLock.lock(); + + if (!this.isValid()) { + throw new IllegalStateException("pinning invalidated frame"); + } + + super.pin(); + } + + /** + * Unpin buffer frame. + */ + @Override + public void unpin() { + super.unpin(); + this.frameLock.unlock(); + } + + /** + * @return whether this frame is valid + */ + @Override + public boolean isValid() { + return this.index >= 0; + } + + /** + * @return whether this frame's page has been freed + */ + private boolean isFreed() { + return this.index < 0 && this.index != INVALID_INDEX; + } + + /** + * Invalidates the frame, flushing it if necessary. + */ + private void invalidate() { + if (this.isValid()) { + this.flush(); + } + this.index = INVALID_INDEX; + this.contents = null; + } + + /** + * Marks the frame as free. + */ + private void setFree() { + if (isFreed()) { + throw new IllegalStateException("cannot free free frame"); + } + int nextFreeIndex = firstFreeIndex; + firstFreeIndex = this.index; + this.index = ~nextFreeIndex; + } + + private void setUsed() { + if (!isFreed()) { + throw new IllegalStateException("cannot unfree used frame"); + } + int index = firstFreeIndex; + firstFreeIndex = ~this.index; + this.index = index; + } + + /** + * @return page number of this frame + */ + @Override + public long getPageNum() { + return this.pageNum; + } + + /** + * Flushes this buffer frame to disk, but does not unload it. + */ + @Override + void flush() { + this.frameLock.lock(); + super.pin(); + try { + if (!this.isValid()) { + return; + } + if (!this.dirty) { + return; + } + if (!this.logPage) { + recoveryManager.pageFlushHook(this.getPageLSN()); + } + BufferManagerImpl.this.diskSpaceManager.writePage(pageNum, contents); + BufferManagerImpl.this.incrementIOs(); + this.dirty = false; + } finally { + super.unpin(); + this.frameLock.unlock(); + } + } + + /** + * Read from the buffer frame. + * @param position position in buffer frame to start reading + * @param num number of bytes to read + * @param buf output buffer + */ + @Override + void readBytes(short position, short num, byte[] buf) { + this.pin(); + try { + if (!this.isValid()) { + throw new IllegalStateException("reading from invalid buffer frame"); + } + System.arraycopy(this.contents, position + dataOffset(), buf, 0, num); + BufferManagerImpl.this.evictionPolicy.hit(this); + } finally { + this.unpin(); + } + } + + /** + * Write to the buffer frame, and mark frame as dirtied. + * @param position position in buffer frame to start writing + * @param num number of bytes to write + * @param buf input buffer + */ + @Override + void writeBytes(short position, short num, byte[] buf) { + this.pin(); + try { + if (!this.isValid()) { + throw new IllegalStateException("writing to invalid buffer frame"); + } + int offset = position + dataOffset(); + TransactionContext transaction = TransactionContext.getTransaction(); + if (transaction != null && !logPage) { + List> changedRanges = getChangedBytes(offset, num, buf); + for (Pair range : changedRanges) { + int start = range.getFirst(); + int len = range.getSecond(); + byte[] before = Arrays.copyOfRange(contents, start + offset, start + offset + len); + byte[] after = Arrays.copyOfRange(buf, start, start + len); + long pageLSN = recoveryManager.logPageWrite(transaction.getTransNum(), pageNum, position, before, + after); + this.setPageLSN(pageLSN); + } + } + System.arraycopy(buf, 0, this.contents, offset, num); + this.dirty = true; + BufferManagerImpl.this.evictionPolicy.hit(this); + } finally { + this.unpin(); + } + } + + /** + * Requests a valid Frame object for the page (if invalid, a new Frame object is returned). + * Page is pinned on return. + */ + @Override + Frame requestValidFrame() { + this.frameLock.lock(); + try { + if (this.isFreed()) { + throw new PageException("page already freed"); + } + if (this.isValid()) { + this.pin(); + return this; + } + return BufferManagerImpl.this.fetchPageFrame(this.pageNum, logPage); + } finally { + this.frameLock.unlock(); + } + } + + @Override + short getEffectivePageSize() { + if (logPage) { + return DiskSpaceManager.PAGE_SIZE; + } else { + return BufferManager.EFFECTIVE_PAGE_SIZE; + } + } + + @Override + long getPageLSN() { + return ByteBuffer.wrap(this.contents).getLong(8); + } + + @Override + public String toString() { + if (index >= 0) { + return "Buffer Frame " + index + ", Page " + pageNum + (isPinned() ? " (pinned)" : ""); + } else if (index == INVALID_INDEX) { + return "Buffer Frame (evicted), Page " + pageNum; + } else { + return "Buffer Frame (freed), next free = " + (~index); + } + } + + /** + * Generates (offset, length) pairs for where buf differs from contents. Merges nearby + * pairs (where nearby is defined as pairs that have fewer than BufferManager.RESERVED_SPACE + * bytes of unmodified data between them). + */ + private List> getChangedBytes(int offset, int num, byte[] buf) { + List> ranges = new ArrayList<>(); + int startIndex = -1; + int skip = -1; + for (int i = 0; i < num; ++i) { + if (buf[i] == contents[offset + i] && startIndex >= 0) { + if (skip > BufferManager.RESERVED_SPACE) { + ranges.add(new Pair<>(startIndex, i - startIndex - skip)); + startIndex = -1; + skip = -1; + } else { + ++skip; + } + } else if (buf[i] != contents[offset + i]) { + if (startIndex < 0) { + startIndex = i; + } + skip = 0; + } + } + if (startIndex >= 0) { + ranges.add(new Pair<>(startIndex, num - startIndex - skip)); + } + return ranges; + } + + void setPageLSN(long pageLSN) { + ByteBuffer.wrap(this.contents).putLong(8, pageLSN); + } + + private short dataOffset() { + if (logPage) { + return 0; + } else { + return BufferManager.RESERVED_SPACE; + } + } + } + + /** + * Creates a new buffer manager. + * + * @param diskSpaceManager the underlying disk space manager + * @param bufferSize size of buffer (in pages) + * @param evictionPolicy eviction policy to use + */ + public BufferManagerImpl(DiskSpaceManager diskSpaceManager, RecoveryManager recoveryManager, + int bufferSize, EvictionPolicy evictionPolicy) { + this.frames = new Frame[bufferSize]; + for (int i = 0; i < bufferSize; ++i) { + this.frames[i] = new Frame(new byte[DiskSpaceManager.PAGE_SIZE], i + 1, false); + } + this.firstFreeIndex = 0; + this.diskSpaceManager = diskSpaceManager; + this.pageToFrame = new HashMap<>(); + this.managerLock = new ReentrantLock(); + this.evictionPolicy = evictionPolicy; + this.recoveryManager = recoveryManager; + } + + @Override + public void close() { + this.managerLock.lock(); + try { + for (Frame frame : this.frames) { + frame.frameLock.lock(); + try { + if (frame.isPinned()) { + throw new IllegalStateException("closing buffer manager but frame still pinned"); + } + if (!frame.isValid()) { + continue; + } + evictionPolicy.cleanup(frame); + frame.invalidate(); + } finally { + frame.frameLock.unlock(); + } + } + } finally { + this.managerLock.unlock(); + } + } + + @Override + public Frame fetchPageFrame(long pageNum, boolean logPage) { + this.managerLock.lock(); + Frame newFrame; + Frame evictedFrame; + // figure out what frame to load data to, and update manager state + try { + if (!this.diskSpaceManager.pageAllocated(pageNum)) { + throw new PageException("page " + pageNum + " not allocated"); + } + if (this.pageToFrame.containsKey(pageNum)) { + newFrame = this.frames[this.pageToFrame.get(pageNum)]; + newFrame.pin(); + return newFrame; + } + // prioritize free frames over eviction + if (this.firstFreeIndex < this.frames.length) { + evictedFrame = this.frames[this.firstFreeIndex]; + evictedFrame.setUsed(); + } else { + evictedFrame = (Frame) evictionPolicy.evict(frames); + this.pageToFrame.remove(evictedFrame.pageNum, evictedFrame.index); + evictionPolicy.cleanup(evictedFrame); + } + int frameIndex = evictedFrame.index; + newFrame = this.frames[frameIndex] = new Frame(evictedFrame.contents, frameIndex, pageNum, logPage); + evictionPolicy.init(newFrame); + + evictedFrame.frameLock.lock(); + newFrame.frameLock.lock(); + + this.pageToFrame.put(pageNum, frameIndex); + } finally { + this.managerLock.unlock(); + } + // flush evicted frame + try { + evictedFrame.invalidate(); + } finally { + evictedFrame.frameLock.unlock(); + } + // read new page into frame + try { + newFrame.pageNum = pageNum; + newFrame.pin(); + BufferManagerImpl.this.diskSpaceManager.readPage(pageNum, newFrame.contents); + this.incrementIOs(); + return newFrame; + } catch (PageException e) { + newFrame.unpin(); + throw e; + } finally { + newFrame.frameLock.unlock(); + } + } + + @Override + public Page fetchPage(LockContext parentContext, long pageNum, boolean logPage) { + return this.frameToPage(parentContext, pageNum, this.fetchPageFrame(pageNum, logPage)); + } + + @Override + public Frame fetchNewPageFrame(int partNum, boolean logPage) { + long pageNum = this.diskSpaceManager.allocPage(partNum); + this.managerLock.lock(); + try { + return fetchPageFrame(pageNum, logPage); + } finally { + this.managerLock.unlock(); + } + } + + @Override + public Page fetchNewPage(LockContext parentContext, int partNum, boolean logPage) { + Frame newFrame = this.fetchNewPageFrame(partNum, logPage); + return this.frameToPage(parentContext, newFrame.getPageNum(), newFrame); + } + + @Override + public void freePage(Page page) { + this.managerLock.lock(); + try { + int frameIndex = this.pageToFrame.get(page.getPageNum()); + Frame frame = this.frames[frameIndex]; + this.pageToFrame.remove(page.getPageNum(), frameIndex); + evictionPolicy.cleanup(frame); + frame.setFree(); + + this.frames[frameIndex] = new Frame(frame); + diskSpaceManager.freePage(page.getPageNum()); + } finally { + this.managerLock.unlock(); + } + } + + @Override + public void freePart(int partNum) { + this.managerLock.lock(); + try { + for (int i = 0; i < frames.length; ++i) { + Frame frame = frames[i]; + if (DiskSpaceManager.getPartNum(frame.pageNum) == partNum) { + this.pageToFrame.remove(frame.getPageNum(), i); + evictionPolicy.cleanup(frame); + frame.setFree(); + + frames[i] = new Frame(frame); + } + } + + diskSpaceManager.freePart(partNum); + } finally { + this.managerLock.unlock(); + } + } + + @Override + public void evict(long pageNum) { + managerLock.lock(); + try { + if (!pageToFrame.containsKey(pageNum)) { + return; + } + evict(pageToFrame.get(pageNum)); + } finally { + managerLock.unlock(); + } + } + + private void evict(int i) { + Frame frame = frames[i]; + frame.frameLock.lock(); + try { + if (frame.isValid() && !frame.isPinned()) { + this.pageToFrame.remove(frame.pageNum, frame.index); + evictionPolicy.cleanup(frame); + + frames[i] = new Frame(frame.contents, this.firstFreeIndex, false); + this.firstFreeIndex = i; + + frame.invalidate(); + } + } finally { + frame.frameLock.unlock(); + } + } + + @Override + public void evictAll() { + for (int i = 0; i < frames.length; ++i) { + evict(i); + } + } + + @Override + public void iterPageNums(BiConsumer process) { + for (Frame frame : frames) { + frame.frameLock.lock(); + try { + if (frame.isValid()) { + process.accept(frame.pageNum, frame.dirty); + } + } finally { + frame.frameLock.unlock(); + } + } + } + + @Override + public long getNumIOs() { + return numIOs; + } + + private void incrementIOs() { + ++numIOs; + } + + /** + * Wraps a frame in a page object. + * @param parentContext parent lock context of the page + * @param pageNum page number + * @param frame frame for the page + * @return page object + */ + private Page frameToPage(LockContext parentContext, long pageNum, Frame frame) { + return new Page(parentContext.childContext(pageNum), frame); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/memory/ClockEvictionPolicy.java b/src/main/java/edu/berkeley/cs186/database/memory/ClockEvictionPolicy.java new file mode 100644 index 0000000..5abf4fb --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/memory/ClockEvictionPolicy.java @@ -0,0 +1,69 @@ +package edu.berkeley.cs186.database.memory; + +/** + * Implementation of clock eviction policy, which works by adding a reference + * bit to each frame, and running the algorithm. + */ +public class ClockEvictionPolicy implements EvictionPolicy { + private int arm; + + private static final Object ACTIVE = true; + // null not false, because default tag (before this class ever sees a frame) is null. + private static final Object INACTIVE = null; + + public ClockEvictionPolicy() { + this.arm = 0; + } + + /** + * Called to initiaize a new buffer frame. + * @param frame new frame to be initialized + */ + @Override + public void init(BufferFrame frame) {} + + /** + * Called when a frame is hit. + * @param frame Frame object that is being read from/written to + */ + @Override + public void hit(BufferFrame frame) { + frame.tag = ACTIVE; + } + + /** + * Called when a frame needs to be evicted. + * @param frames Array of all frames (same length every call) + * @return index of frame to be evicted + * @throws IllegalStateException if everything is pinned + */ + @Override + public BufferFrame evict(BufferFrame[] frames) { + int iters = 0; + // loop around the frames looking for a frame that has bit 0 + // iters is used to ensure that we don't loop forever - after two + // passes through the frames, every frame has bit 0, so if we still haven't + // found a good page to evict, everything is pinned. + while ((frames[this.arm].tag == ACTIVE || frames[this.arm].isPinned()) && + iters < 2 * frames.length) { + frames[this.arm].tag = INACTIVE; + this.arm = (this.arm + 1) % frames.length; + ++iters; + } + if (iters == 2 * frames.length) { + throw new IllegalStateException("cannot evict - everything pinned"); + } + BufferFrame evicted = frames[this.arm]; + this.arm = (this.arm + 1) % frames.length; + return evicted; + } + + /** + * Called when a frame is removed, either because it + * was returned from a call to evict, or because of other constraints + * (e.g. if the page is deleted on disk). + * @param frame frame being removed + */ + @Override + public void cleanup(BufferFrame frame) {} +} diff --git a/src/main/java/edu/berkeley/cs186/database/memory/EvictionPolicy.java b/src/main/java/edu/berkeley/cs186/database/memory/EvictionPolicy.java new file mode 100644 index 0000000..48b6318 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/memory/EvictionPolicy.java @@ -0,0 +1,34 @@ +package edu.berkeley.cs186.database.memory; + +/** + * Interface for eviction policies for the buffer manager. + */ +public interface EvictionPolicy { + /** + * Called to initiaize a new buffer frame. + * @param frame new frame to be initialized + */ + void init(BufferFrame frame); + + /** + * Called when a frame is hit. + * @param frame Frame object that is being read from/written to + */ + void hit(BufferFrame frame); + + /** + * Called when a frame needs to be evicted. + * @param frames Array of all frames (same length every call) + * @return index of frame to be evicted + * @throws IllegalStateException if everything is pinned + */ + BufferFrame evict(BufferFrame[] frames); + + /** + * Called when a frame is removed, either because it + * was returned from a call to evict, or because of other constraints + * (e.g. if the page is deleted on disk). + * @param frame frame being removed + */ + void cleanup(BufferFrame frame); +} diff --git a/src/main/java/edu/berkeley/cs186/database/memory/LRUEvictionPolicy.java b/src/main/java/edu/berkeley/cs186/database/memory/LRUEvictionPolicy.java new file mode 100644 index 0000000..062dec8 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/memory/LRUEvictionPolicy.java @@ -0,0 +1,95 @@ +package edu.berkeley.cs186.database.memory; + +/** + * Implementation of LRU eviction policy, which works by creating a + * doubly-linked list between frames in order of ascending use time. + */ +public class LRUEvictionPolicy implements EvictionPolicy { + private Tag listHead; + private Tag listTail; + + // Doubly-linked list between frames, in order of least to most + // recently used. + private class Tag { + Tag prev = null; + Tag next = null; + BufferFrame cur = null; + + @Override + public String toString() { + String sprev = (prev == null || prev.cur == null) ? "null" : prev.cur.toString(); + String snext = (next == null || next.cur == null) ? "null" : next.cur.toString(); + String scur = cur == null ? "null" : cur.toString(); + return scur + " (prev=" + sprev + ", next=" + snext + ")"; + } + } + + public LRUEvictionPolicy() { + this.listHead = new Tag(); + this.listTail = new Tag(); + this.listHead.next = this.listTail; + this.listTail.prev = this.listHead; + } + + /** + * Called to initiaize a new buffer frame. + * @param frame new frame to be initialized + */ + @Override + public void init(BufferFrame frame) { + Tag frameTag = new Tag(); + frameTag.next = listTail; + frameTag.prev = listTail.prev; + listTail.prev = frameTag; + frameTag.prev.next = frameTag; + frameTag.cur = frame; + frame.tag = frameTag; + } + + /** + * Called when a frame is hit. + * @param frame Frame object that is being read from/written to + */ + @Override + public void hit(BufferFrame frame) { + Tag frameTag = (Tag) frame.tag; + frameTag.prev.next = frameTag.next; + frameTag.next.prev = frameTag.prev; + frameTag.next = this.listTail; + frameTag.prev = this.listTail.prev; + this.listTail.prev.next = frameTag; + this.listTail.prev = frameTag; + } + + /** + * Called when a frame needs to be evicted. + * @param frames Array of all frames (same length every call) + * @return index of frame to be evicted + * @throws IllegalStateException if everything is pinned + */ + @Override + public BufferFrame evict(BufferFrame[] frames) { + Tag frameTag = this.listHead.next; + while (frameTag.cur != null && frameTag.cur.isPinned()) { + frameTag = frameTag.next; + } + if (frameTag.cur == null) { + throw new IllegalStateException("cannot evict anything - everything pinned"); + } + return frameTag.cur; + } + + /** + * Called when a frame is removed, either because it + * was returned from a call to evict, or because of other constraints + * (e.g. if the page is deleted on disk). + * @param frame frame being removed + */ + @Override + public void cleanup(BufferFrame frame) { + Tag frameTag = (Tag) frame.tag; + frameTag.prev.next = frameTag.next; + frameTag.next.prev = frameTag.prev; + frameTag.prev = frameTag.next = frameTag; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/memory/Page.java b/src/main/java/edu/berkeley/cs186/database/memory/Page.java new file mode 100644 index 0000000..174416d --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/memory/Page.java @@ -0,0 +1,245 @@ +package edu.berkeley.cs186.database.memory; + +import edu.berkeley.cs186.database.common.AbstractBuffer; +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.concurrency.*; +import edu.berkeley.cs186.database.io.PageException; + +/** + * Represents a page loaded in memory (as opposed to the buffer frame it's in). Wraps + * around buffer manager frames, and requests the page be loaded into memory as necessary. + */ +public class Page { + // lock context for this page + private LockContext lockContext; + + // buffer manager frame for this page's data (potentially invalidated) + private BufferFrame frame; + + /** + * Create a page handle with the given buffer frame + * + * @param lockContext the lock context + * @param frame the buffer manager frame for this page + */ + Page(LockContext lockContext, BufferFrame frame) { + this.lockContext = lockContext; + this.frame = frame; + } + + /** + * Creates a page handle given another page handle + * + * @param page page handle to copy + */ + protected Page(Page page) { + this.lockContext = page.lockContext; + this.frame = page.frame; + } + + /** + * Disables locking on this page handle. + */ + public void disableLocking() { + this.lockContext = new DummyLockContext(); + } + + /** + * Gets a Buffer object for more convenient access to the page. + * + * @return Buffer object over this page + */ + public Buffer getBuffer() { + return new PageBuffer(); + } + + /** + * Reads num bytes from offset position into buf. + * + * @param position the offset in the page to read from + * @param num the number of bytes to read + * @param buf the buffer to put the bytes into + */ + private void readBytes(int position, int num, byte[] buf) { + if (position < 0 || num < 0) { + throw new PageException("position or num can't be negative"); + } + if (frame.getEffectivePageSize() < position + num) { + throw new PageException("readBytes is out of bounds"); + } + if (buf.length < num) { + throw new PageException("num bytes to read is longer than buffer"); + } + + this.frame.readBytes((short) position, (short) num, buf); + } + + /** + * Read all the bytes in file. + * + * @return a new byte array with all the bytes in the file + */ + private byte[] readBytes() { + byte[] data = new byte[BufferManager.EFFECTIVE_PAGE_SIZE]; + getBuffer().get(data); + return data; + } + + /** + * Write num bytes from buf at offset position. + * + * @param position the offest in the file to write to + * @param num the number of bytes to write + * @param buf the source for the write + */ + private void writeBytes(int position, int num, byte[] buf) { + if (buf.length < num) { + throw new PageException("num bytes to write is longer than buffer"); + } + + if (position < 0 || num < 0) { + throw new PageException("position or num can't be negative"); + } + + if (frame.getEffectivePageSize() < num + position) { + throw new PageException("writeBytes would go out of bounds"); + } + + this.frame.writeBytes((short) position, (short) num, buf); + } + + /** + * Write all the bytes in file. + */ + private void writeBytes(byte[] data) { + getBuffer().put(data); + } + + /** + * Completely wipe (zero out) the page. + */ + public void wipe() { + byte[] zeros = new byte[BufferManager.EFFECTIVE_PAGE_SIZE]; + writeBytes(zeros); + } + + /** + * Force the page to disk. + */ + public void flush() { + this.frame.flush(); + } + + /** + * Loads the page into a frame (if necessary) and pins it. + */ + public void pin() { + this.frame = this.frame.requestValidFrame(); + } + + /** + * Unpins the frame containing this page. Does not flush immediately. + */ + public void unpin() { + this.frame.unpin(); + } + + /** + * @return the virtual page number of this page + */ + public long getPageNum() { + return this.frame.getPageNum(); + } + + /** + * @param pageLSN the new pageLSN of this page - should only be used by recovery + */ + public void setPageLSN(long pageLSN) { + this.frame.setPageLSN(pageLSN); + } + + /** + * @return the pageLSN of this page + */ + public long getPageLSN() { + return this.frame.getPageLSN(); + } + + @Override + public String toString() { + return "Page " + this.frame.getPageNum(); + } + + @Override + public boolean equals(Object b) { + if (!(b instanceof Page)) { + return false; + } + return ((Page) b).getPageNum() == getPageNum(); + } + + /** + * Implementation of Buffer for the page data. All reads/writes ultimately wrap around + * Page#readBytes and Page#writeBytes, which delegates work to the buffer manager. + */ + private class PageBuffer extends AbstractBuffer { + private int offset; + + private PageBuffer() { + this(0, 0); + } + + private PageBuffer(int offset, int position) { + super(position); + this.offset = offset; + } + + /** + * All read operations through the Page object must run through this method. + * + * @param dst destination byte buffer + * @param offset offset into page to start reading + * @param length number of bytes to read + * @return this + */ + @Override + public Buffer get(byte[] dst, int offset, int length) { + // TODO(hw4_part2): locking code here + Page.this.readBytes(this.offset + offset, length, dst); + return this; + } + + /** + * All write operations through the Page object must run through this method. + * + * @param src source byte buffer (to copy to the page) + * @param offset offset into page to start writing + * @param length number of bytes to write + * @return this + */ + @Override + public Buffer put(byte[] src, int offset, int length) { + // TODO(hw4_part2): locking code here + Page.this.writeBytes(this.offset + offset, length, src); + return this; + } + + /** + * Create a new PageBuffer starting at the current offset. + * @return new PageBuffer starting at the current offset + */ + @Override + public Buffer slice() { + return new PageBuffer(offset + position(), 0); + } + + /** + * Create a duplicate PageBuffer object + * @return PageBuffer that is functionally identical to this one + */ + @Override + public Buffer duplicate() { + return new PageBuffer(offset, position()); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/BNLJOperator.java b/src/main/java/edu/berkeley/cs186/database/query/BNLJOperator.java new file mode 100644 index 0000000..2d3ec5c --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/BNLJOperator.java @@ -0,0 +1,166 @@ +package edu.berkeley.cs186.database.query; + +import java.util.*; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.table.Record; + +class BNLJOperator extends JoinOperator { + protected int numBuffers; + + BNLJOperator(QueryOperator leftSource, + QueryOperator rightSource, + String leftColumnName, + String rightColumnName, + TransactionContext transaction) { + super(leftSource, rightSource, leftColumnName, rightColumnName, transaction, JoinType.BNLJ); + + this.numBuffers = transaction.getWorkMemSize(); + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public Iterator iterator() { + return new BNLJIterator(); + } + + @Override + public int estimateIOCost() { + //This method implements the the IO cost estimation of the Block Nested Loop Join + int usableBuffers = numBuffers - 2; + int numLeftPages = getLeftSource().getStats().getNumPages(); + int numRightPages = getRightSource().getStats().getNumPages(); + return ((int) Math.ceil((double) numLeftPages / (double) usableBuffers)) * numRightPages + + numLeftPages; + } + + /** + * BNLJ: Block Nested Loop Join + * See lecture slides. + * + * An implementation of Iterator that provides an iterator interface for this operator. + * + * Word of advice: try to decompose the problem into distinguishable sub-problems. + * This means you'll probably want to add more methods than those given. + */ + private class BNLJIterator extends JoinIterator { + // Iterator over pages of the left relation + private BacktrackingIterator leftIterator; + // Iterator over pages of the right relation + private BacktrackingIterator rightIterator; + // Iterator over records in the current block of left pages + private BacktrackingIterator leftRecordIterator = null; + // Iterator over records in the current right page + private BacktrackingIterator rightRecordIterator = null; + // The current record on the left page + private Record leftRecord = null; + // The next record to return + private Record nextRecord = null; + + private BNLJIterator() { + super(); + + this.leftIterator = BNLJOperator.this.getPageIterator(this.getLeftTableName()); + fetchNextLeftBlock(); + + this.rightIterator = BNLJOperator.this.getPageIterator(this.getRightTableName()); + this.rightIterator.markNext(); + fetchNextRightPage(); + + try { + this.fetchNextRecord(); + } catch (NoSuchElementException e) { + this.nextRecord = null; + } + } + + /** + * Fetch the next non-empty block of B - 2 pages from the left relation. leftRecordIterator + * should be set to a record iterator over the next B - 2 pages of the left relation that + * have a record in them, and leftRecord should be set to the first record in this block. + * + * If there are no more pages in the left relation with records, both leftRecordIterator + * and leftRecord should be set to null. + */ + private void fetchNextLeftBlock() { + // TODO(hw3_part1): implement + } + + /** + * Fetch the next non-empty page from the right relation. rightRecordIterator + * should be set to a record iterator over the next page of the right relation that + * has a record in it. + * + * If there are no more pages in the left relation with records, rightRecordIterator + * should be set to null. + */ + private void fetchNextRightPage() { + // TODO(hw3_part1): implement + } + + /** + * Fetches the next record to return, and sets nextRecord to it. If there are no more + * records to return, a NoSuchElementException should be thrown. + * + * @throws NoSuchElementException if there are no more Records to yield + */ + private void fetchNextRecord() { + // TODO(hw3_part1): implement + } + + /** + * Helper method to create a joined record from a record of the left relation + * and a record of the right relation. + * @param leftRecord Record from the left relation + * @param rightRecord Record from the right relation + * @return joined record + */ + private Record joinRecords(Record leftRecord, Record rightRecord) { + List leftValues = new ArrayList<>(leftRecord.getValues()); + List rightValues = new ArrayList<>(rightRecord.getValues()); + leftValues.addAll(rightValues); + return new Record(leftValues); + } + + /** + * Checks if there are more record(s) to yield + * + * @return true if this iterator has another record to yield, otherwise false + */ + @Override + public boolean hasNext() { + return this.nextRecord != null; + } + + /** + * Yields the next record of this iterator. + * + * @return the next Record + * @throws NoSuchElementException if there are no more Records to yield + */ + @Override + public Record next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + + Record nextRecord = this.nextRecord; + try { + this.fetchNextRecord(); + } catch (NoSuchElementException e) { + this.nextRecord = null; + } + return nextRecord; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/GroupByOperator.java b/src/main/java/edu/berkeley/cs186/database/query/GroupByOperator.java new file mode 100644 index 0000000..46bb435 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/GroupByOperator.java @@ -0,0 +1,155 @@ +package edu.berkeley.cs186.database.query; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.table.MarkerRecord; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.stats.TableStats; + +class GroupByOperator extends QueryOperator { + private int groupByColumnIndex; + private String groupByColumn; + private TransactionContext transaction; + + /** + * Create a new GroupByOperator that pulls from source and groups by groupByColumn. + * + * @param source the source operator of this operator + * @param transaction the transaction containing this operator + * @param groupByColumn the column to group on + */ + GroupByOperator(QueryOperator source, + TransactionContext transaction, + String groupByColumn) { + super(OperatorType.GROUPBY, source); + Schema sourceSchema = this.getSource().getOutputSchema(); + this.transaction = transaction; + this.groupByColumn = this.checkSchemaForColumn(sourceSchema, groupByColumn); + + this.groupByColumnIndex = sourceSchema.getFieldNames().indexOf(this.groupByColumn); + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public boolean isGroupBy() { + return true; + } + + @Override + public Iterator iterator() { + return new GroupByIterator(); + } + + @Override + protected Schema computeSchema() { + return this.getSource().getOutputSchema(); + } + + @Override + public String str() { + return "type: " + this.getType() + + "\ncolumn: " + this.groupByColumn; + } + + /** + * Estimates the table statistics for the result of executing this query operator. + * + * @return estimated TableStats + */ + @Override + public TableStats estimateStats() { + return this.getSource().getStats(); + } + + @Override + public int estimateIOCost() { + return this.getSource().getIOCost(); + } + + /** + * An implementation of Iterator that provides an iterator interface for this operator. + * Returns a marker record between the records of different groups, e.g. + * [group 1 record] [group 1 record] [marker record] [group 2 record] ... + */ + private class GroupByIterator implements Iterator { + private MarkerRecord markerRecord; + private Map hashGroupTempTables; + private int currCount; + private Iterator keyIter; + private Iterator rIter; + + private GroupByIterator() { + Iterator sourceIterator = GroupByOperator.this.getSource().iterator(); + this.markerRecord = MarkerRecord.getMarker(); + this.hashGroupTempTables = new HashMap<>(); + this.currCount = 0; + this.rIter = null; + while (sourceIterator.hasNext()) { + Record record = sourceIterator.next(); + DataBox groupByColumn = record.getValues().get(GroupByOperator.this.groupByColumnIndex); + String tableName; + if (!this.hashGroupTempTables.containsKey(groupByColumn.toString())) { + tableName = GroupByOperator.this.transaction.createTempTable( + GroupByOperator.this.getSource().getOutputSchema()); + this.hashGroupTempTables.put(groupByColumn.toString(), tableName); + } else { + tableName = this.hashGroupTempTables.get(groupByColumn.toString()); + } + GroupByOperator.this.transaction.addRecord(tableName, record.getValues()); + } + this.keyIter = hashGroupTempTables.keySet().iterator(); + } + + /** + * Checks if there are more record(s) to yield + * + * @return true if this iterator has another record to yield, otherwise false + */ + @Override + public boolean hasNext() { + return this.keyIter.hasNext() || (this.rIter != null && this.rIter.hasNext()); + } + + /** + * Yields the next record of this iterator. + * + * @return the next Record + * @throws NoSuchElementException if there are no more Records to yield + */ + @Override + public Record next() { + while (this.hasNext()) { + if (this.rIter != null && this.rIter.hasNext()) { + return this.rIter.next(); + } else if (this.keyIter.hasNext()) { + String key = this.keyIter.next(); + String tableName = this.hashGroupTempTables.get(key); + Iterator prevIter = this.rIter; + try { + this.rIter = GroupByOperator.this.transaction.getRecordIterator(tableName); + } catch (DatabaseException de) { + throw new NoSuchElementException(); + } + if (prevIter != null && ++this.currCount < this.hashGroupTempTables.size()) { + return markerRecord; + } + } + } + throw new NoSuchElementException(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/IndexScanOperator.java b/src/main/java/edu/berkeley/cs186/database/query/IndexScanOperator.java new file mode 100644 index 0000000..0a14459 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/IndexScanOperator.java @@ -0,0 +1,226 @@ +package edu.berkeley.cs186.database.query; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.index.BPlusTree; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.stats.TableStats; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +class IndexScanOperator extends QueryOperator { + private TransactionContext transaction; + private String tableName; + private String columnName; + private PredicateOperator predicate; + private DataBox value; + + private int columnIndex; + + /** + * An index scan operator. + * + * @param transaction the transaction containing this operator + * @param tableName the table to iterate over + * @param columnName the name of the column the index is on + */ + IndexScanOperator(TransactionContext transaction, + String tableName, + String columnName, + PredicateOperator predicate, + DataBox value) { + super(OperatorType.INDEXSCAN); + this.tableName = tableName; + this.transaction = transaction; + this.columnName = columnName; + this.predicate = predicate; + this.value = value; + this.setOutputSchema(this.computeSchema()); + columnName = this.checkSchemaForColumn(this.getOutputSchema(), columnName); + this.columnIndex = this.getOutputSchema().getFieldNames().indexOf(columnName); + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public boolean isIndexScan() { + return true; + } + + @Override + public String str() { + return "type: " + this.getType() + + "\ntable: " + this.tableName + + "\ncolumn: " + this.columnName + + "\noperator: " + this.predicate + + "\nvalue: " + this.value; + } + + /** + * Returns the column name that the index scan is on + * + * @return columnName + */ + public String getColumnName() { + return this.columnName; + } + + /** + * Estimates the table statistics for the result of executing this query operator. + * + * @return estimated TableStats + */ + @Override + public TableStats estimateStats() { + TableStats stats; + + try { + stats = this.transaction.getStats(this.tableName); + } catch (DatabaseException de) { + throw new QueryPlanException(de); + } + + return stats.copyWithPredicate(this.columnIndex, + this.predicate, + this.value); + } + + /** + * Estimates the IO cost of executing this query operator. + * @return estimate IO cost + */ + @Override + public int estimateIOCost() { + int height = transaction.getTreeHeight(tableName, columnName); + int order = transaction.getTreeOrder(tableName, columnName); + TableStats tableStats = transaction.getStats(tableName); + + int count = tableStats.getHistograms().get(columnIndex).copyWithPredicate(predicate, + value).getCount(); + // 2 * order entries/leaf node, but leaf nodes are 50-100% full; we use a fill factor of + // 75% as a rough estimate + return (int) (height + Math.ceil(count / (1.5 * order)) + count); + } + + @Override + public Iterator iterator() { + return new IndexScanIterator(); + } + + @Override + public Schema computeSchema() { + try { + return this.transaction.getFullyQualifiedSchema(this.tableName); + } catch (DatabaseException de) { + throw new QueryPlanException(de); + } + } + + /** + * An implementation of Iterator that provides an iterator interface for this operator. + */ + private class IndexScanIterator implements Iterator { + private Iterator sourceIterator; + private Record nextRecord; + + private IndexScanIterator() { + this.nextRecord = null; + if (IndexScanOperator.this.predicate == PredicateOperator.EQUALS) { + this.sourceIterator = IndexScanOperator.this.transaction.lookupKey( + IndexScanOperator.this.tableName, + IndexScanOperator.this.columnName, + IndexScanOperator.this.value); + } else if (IndexScanOperator.this.predicate == PredicateOperator.LESS_THAN || + IndexScanOperator.this.predicate == PredicateOperator.LESS_THAN_EQUALS) { + this.sourceIterator = IndexScanOperator.this.transaction.sortedScan( + IndexScanOperator.this.tableName, + IndexScanOperator.this.columnName); + } else if (IndexScanOperator.this.predicate == PredicateOperator.GREATER_THAN) { + this.sourceIterator = IndexScanOperator.this.transaction.sortedScanFrom( + IndexScanOperator.this.tableName, + IndexScanOperator.this.columnName, + IndexScanOperator.this.value); + while (this.sourceIterator.hasNext()) { + Record r = this.sourceIterator.next(); + + if (r.getValues().get(IndexScanOperator.this.columnIndex) + .compareTo(IndexScanOperator.this.value) > 0) { + this.nextRecord = r; + break; + } + } + } else if (IndexScanOperator.this.predicate == PredicateOperator.GREATER_THAN_EQUALS) { + this.sourceIterator = IndexScanOperator.this.transaction.sortedScanFrom( + IndexScanOperator.this.tableName, + IndexScanOperator.this.columnName, + IndexScanOperator.this.value); + } + } + + /** + * Checks if there are more record(s) to yield + * + * @return true if this iterator has another record to yield, otherwise false + */ + @Override + public boolean hasNext() { + if (this.nextRecord != null) { + return true; + } + if (IndexScanOperator.this.predicate == PredicateOperator.LESS_THAN) { + if (this.sourceIterator.hasNext()) { + Record r = this.sourceIterator.next(); + if (r.getValues().get(IndexScanOperator.this.columnIndex) + .compareTo(IndexScanOperator.this.value) >= 0) { + return false; + } + this.nextRecord = r; + return true; + } + return false; + } else if (IndexScanOperator.this.predicate == PredicateOperator.LESS_THAN_EQUALS) { + if (this.sourceIterator.hasNext()) { + Record r = this.sourceIterator.next(); + if (r.getValues().get(IndexScanOperator.this.columnIndex) + .compareTo(IndexScanOperator.this.value) > 0) { + return false; + } + this.nextRecord = r; + return true; + } + return false; + } + if (this.sourceIterator.hasNext()) { + this.nextRecord = this.sourceIterator.next(); + return true; + } + return false; + } + + /** + * Yields the next record of this iterator. + * + * @return the next Record + * @throws NoSuchElementException if there are no more Records to yield + */ + @Override + public Record next() { + if (this.hasNext()) { + Record r = this.nextRecord; + this.nextRecord = null; + return r; + } + throw new NoSuchElementException(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/JoinOperator.java b/src/main/java/edu/berkeley/cs186/database/query/JoinOperator.java new file mode 100644 index 0000000..5415eb6 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/JoinOperator.java @@ -0,0 +1,248 @@ +package edu.berkeley.cs186.database.query; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.RecordId; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.stats.TableStats; + +abstract class JoinOperator extends QueryOperator { + enum JoinType { + SNLJ, + PNLJ, + BNLJ, + SORTMERGE + } + + JoinType joinType; + private QueryOperator leftSource; + private QueryOperator rightSource; + private int leftColumnIndex; + private int rightColumnIndex; + private String leftColumnName; + private String rightColumnName; + private TransactionContext transaction; + + /** + * Create a join operator that pulls tuples from leftSource and rightSource. Returns tuples for which + * leftColumnName and rightColumnName are equal. + * + * @param leftSource the left source operator + * @param rightSource the right source operator + * @param leftColumnName the column to join on from leftSource + * @param rightColumnName the column to join on from rightSource + */ + JoinOperator(QueryOperator leftSource, + QueryOperator rightSource, + String leftColumnName, + String rightColumnName, + TransactionContext transaction, + JoinType joinType) { + super(OperatorType.JOIN); + this.joinType = joinType; + this.leftSource = leftSource; + this.rightSource = rightSource; + this.leftColumnName = leftColumnName; + this.rightColumnName = rightColumnName; + this.setOutputSchema(this.computeSchema()); + this.transaction = transaction; + } + + @Override + public boolean isJoin() { + return true; + } + + @Override + public abstract Iterator iterator(); + + @Override + public QueryOperator getSource() { + throw new QueryPlanException("There is no single source for join operators. Please use " + + "getRightSource and getLeftSource and the corresponding set methods."); + } + + QueryOperator getLeftSource() { + return this.leftSource; + } + + QueryOperator getRightSource() { + return this.rightSource; + } + + @Override + public Schema computeSchema() { + Schema leftSchema = this.leftSource.getOutputSchema(); + Schema rightSchema = this.rightSource.getOutputSchema(); + List leftSchemaNames = new ArrayList<>(leftSchema.getFieldNames()); + List rightSchemaNames = new ArrayList<>(rightSchema.getFieldNames()); + this.leftColumnName = this.checkSchemaForColumn(leftSchema, this.leftColumnName); + this.leftColumnIndex = leftSchemaNames.indexOf(leftColumnName); + this.rightColumnName = this.checkSchemaForColumn(rightSchema, this.rightColumnName); + this.rightColumnIndex = rightSchemaNames.indexOf(rightColumnName); + List leftSchemaTypes = new ArrayList<>(leftSchema.getFieldTypes()); + List rightSchemaTypes = new ArrayList<>(rightSchema.getFieldTypes()); + if (!leftSchemaTypes.get(this.leftColumnIndex).getClass().equals(rightSchemaTypes.get( + this.rightColumnIndex).getClass())) { + throw new QueryPlanException("Mismatched types of columns " + leftColumnName + " and " + + rightColumnName + "."); + } + leftSchemaNames.addAll(rightSchemaNames); + leftSchemaTypes.addAll(rightSchemaTypes); + return new Schema(leftSchemaNames, leftSchemaTypes); + } + + @Override + public String str() { + return "type: " + this.joinType + + "\nleftColumn: " + this.leftColumnName + + "\nrightColumn: " + this.rightColumnName; + } + + @Override + public String toString() { + String r = this.str(); + if (this.leftSource != null) { + r += "\n" + ("(left)\n" + this.leftSource.toString()).replaceAll("(?m)^", "\t"); + } + if (this.rightSource != null) { + if (this.leftSource != null) { + r += "\n"; + } + r += "\n" + ("(right)\n" + this.rightSource.toString()).replaceAll("(?m)^", "\t"); + } + return r; + } + + /** + * Estimates the table statistics for the result of executing this query operator. + * + * @return estimated TableStats + */ + @Override + public TableStats estimateStats() { + TableStats leftStats = this.leftSource.getStats(); + TableStats rightStats = this.rightSource.getStats(); + + return leftStats.copyWithJoin(this.leftColumnIndex, + rightStats, + this.rightColumnIndex); + } + + @Override + public abstract int estimateIOCost(); + + public Schema getSchema(String tableName) { + return this.transaction.getSchema(tableName); + } + + public BacktrackingIterator getPageIterator(String tableName) { + return this.transaction.getPageIterator(tableName); + } + + public int getNumEntriesPerPage(String tableName) { + return this.transaction.getNumEntriesPerPage(tableName); + } + + public int getEntrySize(String tableName) { + return this.transaction.getEntrySize(tableName); + } + + public String getLeftColumnName() { + return this.leftColumnName; + } + + public String getRightColumnName() { + return this.rightColumnName; + } + + public TransactionContext getTransaction() { + return this.transaction; + } + + public int getLeftColumnIndex() { + return this.leftColumnIndex; + } + + public int getRightColumnIndex() { + return this.rightColumnIndex; + } + + public Record getRecord(String tableName, RecordId rid) { + return this.transaction.getRecord(tableName, rid); + } + + public BacktrackingIterator getRecordIterator(String tableName) { + return this.transaction.getRecordIterator(tableName); + } + + public BacktrackingIterator getBlockIterator(String tableName, Iterator block, + int maxPages) { + return this.transaction.getBlockIterator(tableName, block, maxPages); + } + + public BacktrackingIterator getTableIterator(String tableName) { + return this.transaction.getRecordIterator(tableName); + } + + public String createTempTable(Schema schema) { + return this.transaction.createTempTable(schema); + } + + public RecordId addRecord(String tableName, List values) { + return this.transaction.addRecord(tableName, values); + } + + public JoinType getJoinType() { + return this.joinType; + } + + /** + * All iterators for subclasses of JoinOperator should subclass from + * JoinIterator; JoinIterator handles creating temporary tables out of the left and right + * input operators. + */ + protected abstract class JoinIterator implements Iterator { + private String leftTableName; + private String rightTableName; + + public JoinIterator() { + if (JoinOperator.this.getLeftSource().isSequentialScan()) { + this.leftTableName = ((SequentialScanOperator) JoinOperator.this.getLeftSource()).getTableName(); + } else { + this.leftTableName = JoinOperator.this.createTempTable( + JoinOperator.this.getLeftSource().getOutputSchema()); + Iterator leftIter = JoinOperator.this.getLeftSource().iterator(); + while (leftIter.hasNext()) { + JoinOperator.this.addRecord(this.leftTableName, leftIter.next().getValues()); + } + } + if (JoinOperator.this.getRightSource().isSequentialScan()) { + this.rightTableName = ((SequentialScanOperator) JoinOperator.this.getRightSource()).getTableName(); + } else { + this.rightTableName = JoinOperator.this.createTempTable( + JoinOperator.this.getRightSource().getOutputSchema()); + Iterator rightIter = JoinOperator.this.getRightSource().iterator(); + while (rightIter.hasNext()) { + JoinOperator.this.addRecord(this.rightTableName, rightIter.next().getValues()); + } + } + } + + protected String getLeftTableName() { + return this.leftTableName; + } + + protected String getRightTableName() { + return this.rightTableName; + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/MaterializeOperator.java b/src/main/java/edu/berkeley/cs186/database/query/MaterializeOperator.java new file mode 100644 index 0000000..7a3e7af --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/MaterializeOperator.java @@ -0,0 +1,33 @@ +package edu.berkeley.cs186.database.query; + +import java.util.Iterator; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.table.Record; + +class MaterializeOperator extends SequentialScanOperator { + /** + * Operator that materializes the source operator into a temporary table immediately, + * and then acts as a sequential scan operator over the temporary table. + * @param source source operator to be materialized + * @param transaction current running transaction + */ + MaterializeOperator(QueryOperator source, + TransactionContext transaction) { + super(OperatorType.MATERIALIZE, transaction, materialize(source, transaction)); + } + + private static String materialize(QueryOperator source, TransactionContext transaction) { + String materializedTableName = transaction.createTempTable(source.getOutputSchema()); + for (Record record : source) { + transaction.addRecord(materializedTableName, record.getValues()); + } + return materializedTableName; + } + + @Override + public String str() { + return "type: " + this.getType(); + } + +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/PNLJOperator.java b/src/main/java/edu/berkeley/cs186/database/query/PNLJOperator.java new file mode 100644 index 0000000..6e93a16 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/PNLJOperator.java @@ -0,0 +1,20 @@ +package edu.berkeley.cs186.database.query; + +import edu.berkeley.cs186.database.TransactionContext; + +class PNLJOperator extends BNLJOperator { + PNLJOperator(QueryOperator leftSource, + QueryOperator rightSource, + String leftColumnName, + String rightColumnName, + TransactionContext transaction) { + super(leftSource, + rightSource, + leftColumnName, + rightColumnName, + transaction); + + joinType = JoinType.PNLJ; + numBuffers = 3; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/ProjectOperator.java b/src/main/java/edu/berkeley/cs186/database/query/ProjectOperator.java new file mode 100644 index 0000000..a892a13 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/ProjectOperator.java @@ -0,0 +1,326 @@ +package edu.berkeley.cs186.database.query; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.databox.TypeId; +import edu.berkeley.cs186.database.table.MarkerRecord; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.stats.TableStats; + +class ProjectOperator extends QueryOperator { + private List columns; + private List indices; + private boolean hasCount; + private int averageColumnIndex; + private int sumColumnIndex; + private boolean hasAggregate; + private int countValue; + private double sumValue; + private double averageSumValue; + private int averageCountValue; + private String sumColumn; + private String averageColumn; + private boolean sumIsFloat; + + /** + * Creates a new ProjectOperator that reads tuples from source and filters out columns. Optionally + * computers an aggregate if it is specified. + * + * @param source + * @param columns + * @param count + * @param averageColumn + * @param sumColumn + */ + ProjectOperator(QueryOperator source, + List columns, + boolean count, + String averageColumn, + String sumColumn) { + super(OperatorType.PROJECT); + this.columns = columns; + this.indices = new ArrayList<>(); + this.countValue = 0; + this.sumValue = 0; + this.averageCountValue = 0; + this.averageSumValue = 0; + this.averageColumnIndex = -1; + this.sumColumnIndex = -1; + this.sumColumn = sumColumn; + this.averageColumn = averageColumn; + this.hasCount = count; + this.hasAggregate = this.hasCount || averageColumn != null || sumColumn != null; + + // NOTE: Don't need to explicitly set the output schema because setting the source recomputes + // the schema for the query optimization case. + this.setSource(source); + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public boolean isProject() { + return true; + } + + @Override + protected Schema computeSchema() { + // check to make sure that the source operator is giving us columns that we project + Schema sourceSchema = this.getSource().getOutputSchema(); + List sourceColumnNames = new ArrayList<>(sourceSchema.getFieldNames()); + List sourceColumnTypes = new ArrayList<>(sourceSchema.getFieldTypes()); + List columnTypes = new ArrayList<>(); + for (String columnName : this.columns) { + columnName = this.checkSchemaForColumn(sourceSchema, columnName); + int sourceColumnIndex = sourceColumnNames.indexOf(columnName); + columnTypes.add(sourceColumnTypes.get(sourceColumnIndex)); + this.indices.add(sourceColumnIndex); + } + if (this.sumColumn != null) { + this.sumColumn = this.checkSchemaForColumn(sourceSchema, this.sumColumn); + this.sumColumnIndex = sourceColumnNames.indexOf(this.sumColumn); + if (!(sourceColumnTypes.get(this.sumColumnIndex).getTypeId() == TypeId.INT) && + !(sourceColumnTypes.get(this.sumColumnIndex).getTypeId() == TypeId.FLOAT)) { + throw new QueryPlanException("Cannot compute sum over a non-integer column: " + this.sumColumn + + "."); + } + } + if (this.averageColumn != null) { + this.averageColumn = this.checkSchemaForColumn(sourceSchema, this.averageColumn); + this.averageColumnIndex = sourceColumnNames.indexOf(this.averageColumn); + if (!(sourceColumnTypes.get(this.averageColumnIndex).getTypeId() == TypeId.INT) && + !(sourceColumnTypes.get(this.sumColumnIndex).getTypeId() == TypeId.FLOAT)) { + throw new QueryPlanException("Cannot compute sum over a non-integer column: " + this.averageColumn + + "."); + } + } + + // make sure we add the correct columns to the output schema if we have aggregates in the + // projection + if (this.hasAggregate) { + if (this.hasCount) { + this.columns.add("countAgg"); + columnTypes.add(Type.intType()); + } + if (this.sumColumn != null) { + this.columns.add("sumAgg"); + if (sourceColumnTypes.get(this.sumColumnIndex).getTypeId() == TypeId.INT) { + columnTypes.add(Type.intType()); + this.sumIsFloat = false; + } else { + columnTypes.add(Type.floatType()); + this.sumIsFloat = true; + } + } + if (this.averageColumn != null) { + this.columns.add("averageAgg"); + columnTypes.add(Type.floatType()); + } + } + return new Schema(this.columns, columnTypes); + } + + @Override + public Iterator iterator() { return new ProjectIterator(); } + + private void addToCount() { + this.countValue++; + } + + private int getAndResetCount() { + int result = this.countValue; + this.countValue = 0; + return result; + } + + private void addToSum(Record record) { + if (this.sumIsFloat) { + this.sumValue += record.getValues().get(this.sumColumnIndex).getFloat(); + } else { + this.sumValue += record.getValues().get(this.sumColumnIndex).getInt(); + } + } + + private double getAndResetSum() { + double result = this.sumValue; + this.sumValue = 0; + return result; + } + + private void addToAverage(Record record) { + this.averageCountValue++; + this.averageSumValue += record.getValues().get(this.averageColumnIndex).getInt(); + } + + private double getAndResetAverage() { + if (this.averageCountValue == 0) { + return 0f; + } + + double result = this.averageSumValue / this.averageCountValue; + this.averageSumValue = 0; + this.averageCountValue = 0; + return result; + } + + @Override + public String str() { + return "type: " + this.getType() + + "\ncolumns: " + this.columns; + } + + /** + * Estimates the table statistics for the result of executing this query operator. + * + * @return estimated TableStats + */ + @Override + public TableStats estimateStats() { + return this.getSource().getStats(); + } + + @Override + public int estimateIOCost() { + return this.getSource().getIOCost(); + } + + /** + * An implementation of Iterator that provides an iterator interface for this operator. + */ + private class ProjectIterator implements Iterator { + private Iterator sourceIterator; + private MarkerRecord markerRecord; + private boolean prevWasMarker; + private List baseValues; + + private ProjectIterator() { + this.sourceIterator = ProjectOperator.this.getSource().iterator(); + this.markerRecord = MarkerRecord.getMarker(); + this.prevWasMarker = true; + this.baseValues = new ArrayList<>(); + } + + /** + * Checks if there are more record(s) to yield + * + * @return true if this iterator has another record to yield, otherwise false + */ + @Override + public boolean hasNext() { + return this.sourceIterator.hasNext(); + } + + /** + * Yields the next record of this iterator. + * + * @return the next Record + * @throws NoSuchElementException if there are no more Records to yield + */ + @Override + public Record next() { + if (this.hasNext()) { + if (ProjectOperator.this.hasAggregate) { + while (this.sourceIterator.hasNext()) { + Record r = this.sourceIterator.next(); + List recordValues = r.getValues(); + + // if the record is a MarkerRecord, that means we reached the end of a group... we reset + // the aggregates and add the appropriate new record to the new Records + if (r == this.markerRecord) { + if (ProjectOperator.this.hasCount) { + int count = ProjectOperator.this.getAndResetCount(); + this.baseValues.add(new IntDataBox(count)); + } + if (ProjectOperator.this.sumColumnIndex != -1) { + double sum = ProjectOperator.this.getAndResetSum(); + + if (ProjectOperator.this.sumIsFloat) { + this.baseValues.add(new FloatDataBox((float) sum)); + } else { + this.baseValues.add(new IntDataBox((int) sum)); + } + } + if (ProjectOperator.this.averageColumnIndex != -1) { + double average = (float) ProjectOperator.this.getAndResetAverage(); + this.baseValues.add(new FloatDataBox((float) average)); + } + // record that we just saw a marker record + this.prevWasMarker = true; + return new Record(this.baseValues); + } else { + // if the previous record was a marker (or for the first record) we have to get the relevant + // fields out of the record + if (this.prevWasMarker) { + this.baseValues = new ArrayList<>(); + for (int index : ProjectOperator.this.indices) { + this.baseValues.add(recordValues.get(index)); + } + this.prevWasMarker = false; + } + if (ProjectOperator.this.hasCount) { + ProjectOperator.this.addToCount(); + } + if (ProjectOperator.this.sumColumnIndex != -1) { + ProjectOperator.this.addToSum(r); + } + if (ProjectOperator.this.averageColumnIndex != -1) { + ProjectOperator.this.addToAverage(r); + } + } + } + + // at the very end, we need to make sure we add all the aggregated records to the result + // either because there was no group by or to add the last group we saw + if (ProjectOperator.this.hasCount) { + int count = ProjectOperator.this.getAndResetCount(); + this.baseValues.add(new IntDataBox(count)); + } + if (ProjectOperator.this.sumColumnIndex != -1) { + double sum = ProjectOperator.this.getAndResetSum(); + + if (ProjectOperator.this.sumIsFloat) { + this.baseValues.add(new FloatDataBox((float) sum)); + } else { + this.baseValues.add(new IntDataBox((int) sum)); + } + } + if (ProjectOperator.this.averageColumnIndex != -1) { + double average = ProjectOperator.this.getAndResetAverage(); + this.baseValues.add(new FloatDataBox((float) average)); + } + return new Record(this.baseValues); + } else { + Record r = this.sourceIterator.next(); + List recordValues = r.getValues(); + List newValues = new ArrayList<>(); + + // if there is a marker record (in the case we're projecting from a group by), we simply + // leave the marker records in + if (r == this.markerRecord) { + return markerRecord; + } else { + for (int index : ProjectOperator.this.indices) { + newValues.add(recordValues.get(index)); + } + return new Record(newValues); + } + } + } + throw new NoSuchElementException(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/QueryOperator.java b/src/main/java/edu/berkeley/cs186/database/query/QueryOperator.java new file mode 100644 index 0000000..86c73d3 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/QueryOperator.java @@ -0,0 +1,186 @@ +package edu.berkeley.cs186.database.query; + +import java.util.Iterator; +import java.util.List; + +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.stats.TableStats; + +public abstract class QueryOperator implements Iterable { + private QueryOperator source; + private QueryOperator destination; + private Schema operatorSchema; + protected TableStats stats; + protected int cost; + + public enum OperatorType { + JOIN, + PROJECT, + SELECT, + GROUPBY, + SEQSCAN, + INDEXSCAN, + MATERIALIZE, + } + + private OperatorType type; + + public QueryOperator(OperatorType type) { + this.type = type; + this.source = null; + this.operatorSchema = null; + this.destination = null; + } + + protected QueryOperator(OperatorType type, QueryOperator source) { + this.source = source; + this.type = type; + this.operatorSchema = this.computeSchema(); + this.destination = null; + } + + public OperatorType getType() { + return this.type; + } + + public boolean isJoin() { + return this.type.equals(OperatorType.JOIN); + } + + public boolean isSelect() { + return this.type.equals(OperatorType.SELECT); + } + + public boolean isProject() { + return this.type.equals(OperatorType.PROJECT); + } + + public boolean isGroupBy() { + return this.type.equals(OperatorType.GROUPBY); + } + + public boolean isSequentialScan() { + return this.type.equals(OperatorType.SEQSCAN); + } + + public boolean isIndexScan() { + return this.type.equals(OperatorType.INDEXSCAN); + } + + public QueryOperator getSource() { + return this.source; + } + + public QueryOperator getDestination() { + return this.destination; + } + + void setSource(QueryOperator source) { + this.source = source; + this.operatorSchema = this.computeSchema(); + } + + public void setDestination(QueryOperator destination) { + this.destination = destination; + } + + Schema getOutputSchema() { + return this.operatorSchema; + } + + void setOutputSchema(Schema schema) { + this.operatorSchema = schema; + } + + protected abstract Schema computeSchema(); + + public Iterator execute() { + return iterator(); + } + + public abstract Iterator iterator(); + + /** + * Utility method that checks to see if a column is found in a schema using dot notation. + * + * @param fromSchema the schema to search in + * @param specified the column name to search for + * @return + */ + public boolean checkColumnNameEquality(String fromSchema, String specified) { + if (fromSchema.equals(specified)) { + return true; + } + if (!specified.contains(".")) { + String schemaColName = fromSchema; + if (fromSchema.contains(".")) { + String[] splits = fromSchema.split("\\."); + schemaColName = splits[1]; + } + + return schemaColName.equals(specified); + } + return false; + } + + /** + * Utility method to determine whether or not a specified column name is valid with a given schema. + * + * @param schema + * @param columnName + */ + public String checkSchemaForColumn(Schema schema, String columnName) { + List schemaColumnNames = schema.getFieldNames(); + boolean found = false; + String foundName = null; + for (String sourceColumnName : schemaColumnNames) { + if (this.checkColumnNameEquality(sourceColumnName, columnName)) { + if (found) { + throw new QueryPlanException("Column " + columnName + " specified twice without disambiguation."); + } + found = true; + foundName = sourceColumnName; + } + } + if (!found) { + throw new QueryPlanException("No column " + columnName + " found."); + } + return foundName; + } + + public String str() { + return "type: " + this.getType(); + } + + public String toString() { + String r = this.str(); + if (this.source != null) { + r += "\n" + this.source.toString().replaceAll("(?m)^", "\t"); + } + return r; + } + + /** + * Estimates the table statistics for the result of executing this query operator. + * + * @return estimated TableStats + */ + protected abstract TableStats estimateStats(); + + /** + * Estimates the IO cost of executing this query operator. + * + * @return estimated number of IO's performed + */ + public abstract int estimateIOCost(); + + public TableStats getStats() { + return this.stats; + } + + public int getIOCost() { + return this.cost; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/QueryPlan.java b/src/main/java/edu/berkeley/cs186/database/query/QueryPlan.java new file mode 100644 index 0000000..0ac1c52 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/QueryPlan.java @@ -0,0 +1,527 @@ +package edu.berkeley.cs186.database.query; + +import java.util.*; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; + +/** + * QueryPlan provides a set of functions to generate simple queries. Calling the methods corresponding + * to SQL syntax stores the information in the QueryPlan, and calling execute generates and executes + * a QueryPlan DAG. + */ +public class QueryPlan { + private TransactionContext transaction; + private QueryOperator finalOperator; + private String startTableName; + + private List joinTableNames; + private List joinLeftColumnNames; + private List joinRightColumnNames; + private List selectColumnNames; + private List selectOperators; + private List selectDataBoxes; + private List projectColumns; + private Map aliases; + private String groupByColumn; + private boolean hasCount; + private String averageColumnName; + private String sumColumnName; + + /** + * Creates a new QueryPlan within transaction. The base table is startTableName. + * + * @param transaction the transaction containing this query + * @param startTableName the source table for this query + */ + public QueryPlan(TransactionContext transaction, String startTableName) { + this(transaction, startTableName, startTableName); + } + + /** + * Creates a new QueryPlan within transaction. The base table is startTableName, + * aliased to aliasTableName. + * + * @param transaction the transaction containing this query + * @param startTableName the source table for this query + * @param aliasTableName the alias for the source table + */ + public QueryPlan(TransactionContext transaction, String startTableName, String aliasTableName) { + this.transaction = transaction; + this.startTableName = aliasTableName; + + this.projectColumns = new ArrayList<>(); + this.joinTableNames = new ArrayList<>(); + this.joinLeftColumnNames = new ArrayList<>(); + this.joinRightColumnNames = new ArrayList<>(); + + this.selectColumnNames = new ArrayList<>(); + this.selectOperators = new ArrayList<>(); + this.selectDataBoxes = new ArrayList<>(); + + this.aliases = new HashMap<>(); + this.aliases.put(aliasTableName, startTableName); + + this.hasCount = false; + this.averageColumnName = null; + this.sumColumnName = null; + + this.groupByColumn = null; + + this.finalOperator = null; + + this.transaction.setAliasMap(this.aliases); + } + + public QueryOperator getFinalOperator() { + return this.finalOperator; + } + + /** + * Add a project operator to the QueryPlan with a list of column names. Can only specify one set + * of projections. + * + * @param columnNames the columns to project + */ + public void project(List columnNames) { + if (!this.projectColumns.isEmpty()) { + throw new QueryPlanException("Cannot add more than one project operator to this query."); + } + + if (columnNames.isEmpty()) { + throw new QueryPlanException("Cannot project no columns."); + } + + this.projectColumns = new ArrayList<>(columnNames); + } + + /** + * Add a select operator. Only returns columns in which the column fulfills the predicate relative + * to value. + * + * @param column the column to specify the predicate on + * @param comparison the comparator + * @param value the value to compare against + */ + public void select(String column, PredicateOperator comparison, + DataBox value) { + this.selectColumnNames.add(column); + this.selectOperators.add(comparison); + this.selectDataBoxes.add(value); + } + + /** + * Set the group by column for this query. + * + * @param column the column to group by + */ + public void groupBy(String column) { + this.groupByColumn = column; + } + + /** + * Add a count aggregate to this query. Only can specify count(*). + */ + public void count() { + this.hasCount = true; + } + + /** + * Add an average on column. Can only average over integer or float columns. + * + * @param column the column to average + */ + public void average(String column) { + this.averageColumnName = column; + } + + /** + * Add a sum on column. Can only sum integer or float columns + * + * @param column the column to sum + */ + public void sum(String column) { + this.sumColumnName = column; + } + + /** + * Join the leftColumnName column of the existing queryplan against the rightColumnName column + * of tableName. + * + * @param tableName the table to join against + * @param leftColumnName the join column in the existing QueryPlan + * @param rightColumnName the join column in tableName + */ + public void join(String tableName, String leftColumnName, String rightColumnName) { + join(tableName, tableName, leftColumnName, rightColumnName); + } + + /** + * Join the leftColumnName column of the existing queryplan against the rightColumnName column + * of tableName, aliased as aliasTableName. + * + * @param tableName the table to join against + * @param aliasTableName alias of table to join against + * @param leftColumnName the join column in the existing QueryPlan + * @param rightColumnName the join column in tableName + */ + public void join(String tableName, String aliasTableName, String leftColumnName, + String rightColumnName) { + if (this.aliases.containsKey(aliasTableName)) { + throw new QueryPlanException("table/alias " + aliasTableName + " already in use"); + } + this.joinTableNames.add(aliasTableName); + this.aliases.put(aliasTableName, tableName); + this.joinLeftColumnNames.add(leftColumnName); + this.joinRightColumnNames.add(rightColumnName); + this.transaction.setAliasMap(this.aliases); + } + + //Returns a 2-array of table name, column name + private String [] getJoinLeftColumnNameByIndex(int i) { + return this.joinLeftColumnNames.get(i).split("\\."); + } + + //Returns a 2-array of table name, column name + private String [] getJoinRightColumnNameByIndex(int i) { + return this.joinRightColumnNames.get(i).split("\\."); + } + + /** + * Generates a naive QueryPlan in which all joins are at the bottom of the DAG followed by all select + * predicates, an optional group by operator, and a set of projects (in that order). + * + * @return an iterator of records that is the result of this query + */ + public Iterator executeNaive() { + this.transaction.setAliasMap(this.aliases); + try { + String indexColumn = this.checkIndexEligible(); + + if (indexColumn != null) { + this.generateIndexPlan(indexColumn); + } else { + // start off with the start table scan as the source + this.finalOperator = new SequentialScanOperator(this.transaction, this.startTableName); + + this.addJoins(); + this.addSelects(); + this.addGroupBy(); + this.addProjects(); + } + + return this.finalOperator.execute(); + } finally { + this.transaction.clearAliasMap(); + } + } + + /** + * Generates an optimal QueryPlan based on the System R cost-based query optimizer. + * + * @return an iterator of records that is the result of this query + */ + public Iterator execute() { + // TODO(hw3_part2): implement + + // Pass 1: Iterate through all single tables. For each single table, find + // the lowest cost QueryOperator to access that table. Construct a mapping + // of each table name to its lowest cost operator. + + // Pass i: On each pass, use the results from the previous pass to find the + // lowest cost joins with each single table. Repeat until all tables have + // been joined. + + // Get the lowest cost operator from the last pass, add GROUP BY and SELECT + // operators, and return an iterator on the final operator + + return this.executeNaive(); // TODO(hw3_part2): Replace this!!! Allows you to test intermediate functionality + } + + /** + * Gets all SELECT predicates for which there exists an index on the column + * referenced in that predicate for the given table. + * + * @return an ArrayList of SELECT predicates + */ + private List getEligibleIndexColumns(String table) { + List selectIndices = new ArrayList<>(); + + for (int i = 0; i < this.selectColumnNames.size(); i++) { + String column = this.selectColumnNames.get(i); + + if (this.transaction.indexExists(table, column) && + this.selectOperators.get(i) != PredicateOperator.NOT_EQUALS) { + selectIndices.add(i); + } + } + + return selectIndices; + } + + /** + * Gets all columns for which there exists an index for that table + * + * @return an ArrayList of column names + */ + private List getAllIndexColumns(String table) { + List indexColumns = new ArrayList<>(); + + Schema schema = this.transaction.getSchema(table); + List columnNames = schema.getFieldNames(); + + for (String column : columnNames) { + if (this.transaction.indexExists(table, column)) { + indexColumns.add(table + "." + column); + } + } + + return indexColumns; + } + + /** + * Applies all eligible SELECT predicates to a given source, except for the + * predicate at index except. The purpose of except is because there might + * be one SELECT predicate that was already used for an index scan, so no + * point applying it again. A SELECT predicate is represented as elements of + * this.selectColumnNames, this.selectOperators, and this.selectDataBoxes that + * correspond to the same index of these lists. + * + * @return a new QueryOperator after SELECT has been applied + */ + private QueryOperator addEligibleSelections(QueryOperator source, int except) { + for (int i = 0; i < this.selectColumnNames.size(); i++) { + if (i == except) { + continue; + } + + PredicateOperator curPred = this.selectOperators.get(i); + DataBox curValue = this.selectDataBoxes.get(i); + try { + String colName = source.checkSchemaForColumn(source.getOutputSchema(), selectColumnNames.get(i)); + source = new SelectOperator(source, colName, curPred, curValue); + } catch (QueryPlanException err) { + /* do nothing */ + } + } + + return source; + } + + /** + * Finds the lowest cost QueryOperator that scans the given table. First + * determine the cost of a sequential scan for the given table. Then for every index that can be + * used on that table, determine the cost of an index scan. Keep track of + * the minimum cost operation. Then push down eligible projects (SELECT + * predicates). If an index scan was chosen, exclude that SELECT predicate when + * pushing down selects. This method will be called during the first pass of the search + * algorithm to determine the most efficient way to access each single table. + * + * @return a QueryOperator that has the lowest cost of scanning the given table which is + * either a SequentialScanOperator or an IndexScanOperator nested within any possible + * pushed down select operators + */ + QueryOperator minCostSingleAccess(String table) { + QueryOperator minOp = null; + + // Find the cost of a sequential scan of the table + // minOp = new SequentialScanOperator(this.transaction, table); + + // TODO(hw3_part2): implement + + // 1. Find the cost of a sequential scan of the table + + // 2. For each eligible index column, find the cost of an index scan of the + // table and retain the lowest cost operator + + // 3. Push down SELECT predicates that apply to this table and that were not + // used for an index scan + + return minOp; + } + + /** + * Given a join condition between an outer relation represented by leftOp + * and an inner relation represented by rightOp, find the lowest cost join + * operator out of all the possible join types in JoinOperator.JoinType. + * + * @return lowest cost join QueryOperator between the input operators + */ + private QueryOperator minCostJoinType(QueryOperator leftOp, + QueryOperator rightOp, + String leftColumn, + String rightColumn) { + QueryOperator minOp = null; + + int minCost = Integer.MAX_VALUE; + List allJoins = new ArrayList<>(); + allJoins.add(new SNLJOperator(leftOp, rightOp, leftColumn, rightColumn, this.transaction)); + allJoins.add(new BNLJOperator(leftOp, rightOp, leftColumn, rightColumn, this.transaction)); + + for (QueryOperator join : allJoins) { + int joinCost = join.estimateIOCost(); + if (joinCost < minCost) { + minOp = join; + minCost = joinCost; + } + } + return minOp; + } + + /** + * Iterate through all table sets in the previous pass of the search. For each + * table set, check each join predicate to see if there is a valid join + * condition with a new table. If so, check the cost of each type of join and + * keep the minimum cost join. Construct and return a mapping of each set of + * table names being joined to its lowest cost join operator. A join predicate + * is represented as elements of this.joinTableNames, this.joinLeftColumnNames, + * and this.joinRightColumnNames that correspond to the same index of these lists. + * + * @return a mapping of table names to a join QueryOperator + */ + Map minCostJoins(Map prevMap, + Map pass1Map) { + Map map = new HashMap<>(); + + // TODO(hw3_part2): implement + + //We provide a basic description of the logic you have to implement + + //Input: prevMap (maps a set of tables to a query operator--the operator that joins the set) + //Input: pass1Map (each set is a singleton with one table and single table access query operator) + + //FOR EACH set of tables in prevMap: + + //FOR EACH join condition listed in the query + + //get the left side and the right side (table name and column) + + /* + * Case 1. Set contains left table but not right, use pass1Map to + * fetch the right operator to access the rightTable + * + * Case 2. Set contains right table but not left, use pass1Map to + * fetch the right operator to access the leftTable. + * + * Case 3. Set contains neither or both the left table or right table (continue loop) + * + * --- Then given the operator, use minCostJoinType to calculate the cheapest join with that + * and the previously joined tables. + */ + + return map; + } + + /** + * Finds the lowest cost QueryOperator in the given mapping. A mapping is + * generated on each pass of the search algorithm, and relates a set of tables + * to the lowest cost QueryOperator accessing those tables. This method is + * called at the end of the search algorithm after all passes have been + * processed. + * + * @return a QueryOperator in the given mapping + */ + private QueryOperator minCostOperator(Map map) { + QueryOperator minOp = null; + QueryOperator newOp; + int minCost = Integer.MAX_VALUE; + int newCost; + for (Set tables : map.keySet()) { + newOp = map.get(tables); + newCost = newOp.getIOCost(); + if (newCost < minCost) { + minOp = newOp; + minCost = newCost; + } + } + return minOp; + } + + private String checkIndexEligible() { + if (this.selectColumnNames.size() > 0 + && this.groupByColumn == null + && this.joinTableNames.size() == 0) { + int index = 0; + for (String column : selectColumnNames) { + if (this.transaction.indexExists(this.startTableName, column)) { + if (this.selectOperators.get(index) != PredicateOperator.NOT_EQUALS) { + return column; + } + } + + index++; + } + } + + return null; + } + + private void generateIndexPlan(String indexColumn) { + int selectIndex = this.selectColumnNames.indexOf(indexColumn); + PredicateOperator operator = this.selectOperators.get(selectIndex); + DataBox value = this.selectDataBoxes.get(selectIndex); + + this.finalOperator = new IndexScanOperator(this.transaction, this.startTableName, indexColumn, + operator, + value); + + this.selectColumnNames.remove(selectIndex); + this.selectOperators.remove(selectIndex); + this.selectDataBoxes.remove(selectIndex); + + this.addSelects(); + this.addProjects(); + } + + private void addJoins() { + int index = 0; + + for (String joinTable : this.joinTableNames) { + SequentialScanOperator scanOperator = new SequentialScanOperator(this.transaction, joinTable); + + this.finalOperator = new SNLJOperator(finalOperator, scanOperator, + this.joinLeftColumnNames.get(index), this.joinRightColumnNames.get(index), + this.transaction); + + index++; + } + } + + private void addSelects() { + int index = 0; + + for (String selectColumn : this.selectColumnNames) { + PredicateOperator operator = this.selectOperators.get(index); + DataBox value = this.selectDataBoxes.get(index); + + this.finalOperator = new SelectOperator(this.finalOperator, selectColumn, + operator, value); + + index++; + } + } + + private void addGroupBy() { + if (this.groupByColumn != null) { + if (this.projectColumns.size() > 2 || (this.projectColumns.size() == 1 && + !this.projectColumns.get(0).equals(this.groupByColumn))) { + throw new QueryPlanException("Can only project columns specified in the GROUP BY clause."); + } + + this.finalOperator = new GroupByOperator(this.finalOperator, this.transaction, + this.groupByColumn); + } + } + + private void addProjects() { + if (!this.projectColumns.isEmpty() || this.hasCount || this.sumColumnName != null + || this.averageColumnName != null) { + this.finalOperator = new ProjectOperator(this.finalOperator, this.projectColumns, + this.hasCount, this.averageColumnName, this.sumColumnName); + } + } + +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/QueryPlanException.java b/src/main/java/edu/berkeley/cs186/database/query/QueryPlanException.java new file mode 100644 index 0000000..612a53e --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/QueryPlanException.java @@ -0,0 +1,19 @@ +package edu.berkeley.cs186.database.query; + +public class QueryPlanException extends RuntimeException { + private String message; + + public QueryPlanException(String message) { + this.message = message; + } + + public QueryPlanException(Exception e) { + this.message = e.getClass().toString() + ": " + e.getMessage(); + } + + @Override + public String getMessage() { + return this.message; + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/query/SNLJOperator.java b/src/main/java/edu/berkeley/cs186/database/query/SNLJOperator.java new file mode 100644 index 0000000..d74f6ce --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/SNLJOperator.java @@ -0,0 +1,160 @@ +package edu.berkeley.cs186.database.query; + +import java.util.*; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.table.Record; + +public class SNLJOperator extends JoinOperator { + public SNLJOperator(QueryOperator leftSource, + QueryOperator rightSource, + String leftColumnName, + String rightColumnName, + TransactionContext transaction) { + super(leftSource, + rightSource, + leftColumnName, + rightColumnName, + transaction, + JoinType.SNLJ); + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public Iterator iterator() { + return new SNLJIterator(); + } + + @Override + public int estimateIOCost() { + int numLeftRecords = getLeftSource().getStats().getNumRecords(); + + int numRightPages = getRightSource().getStats().getNumPages(); + int numLeftPages = getLeftSource().getStats().getNumPages(); + + return numLeftRecords * numRightPages + numLeftPages; + } + + /** + * An implementation of Iterator that provides an iterator interface for this operator. + * Note that the left table is the "outer" loop and the right table is the "inner" loop. + */ + private class SNLJIterator extends JoinIterator { + private BacktrackingIterator leftIterator; + private BacktrackingIterator rightIterator; + private Record leftRecord; + private Record rightRecord; + private Record nextRecord; + + public SNLJIterator() { + super(); + this.rightIterator = SNLJOperator.this.getRecordIterator(this.getRightTableName()); + this.leftIterator = SNLJOperator.this.getRecordIterator(this.getLeftTableName()); + + this.nextRecord = null; + + this.leftRecord = leftIterator.hasNext() ? leftIterator.next() : null; + this.rightRecord = rightIterator.hasNext() ? rightIterator.next() : null; + + // We mark the first record so we can reset to it when we advance the left record. + if (rightRecord != null) { + rightIterator.markPrev(); + } else { return; } + + try { + fetchNextRecord(); + } catch (NoSuchElementException e) { + this.nextRecord = null; + } + } + + /** + * After this method is called, rightRecord will contain the first record in the rightSource. + * There is always a first record. If there were no first records (empty rightSource) + * then the code would not have made it this far. See line 69. + */ + private void resetRightRecord() { + this.rightIterator.reset(); + assert(rightIterator.hasNext()); + rightRecord = rightIterator.next(); + } + + /** + * Advances the left record + * + * The thrown exception means we're done: there is no next record + * It causes this.fetchNextRecord (the caller) to hand control to its caller. + */ + private void nextLeftRecord() { + if (!leftIterator.hasNext()) { throw new NoSuchElementException("All Done!"); } + leftRecord = leftIterator.next(); + } + + /** + * Pre-fetches what will be the next record, and puts it in this.nextRecord. + * Pre-fetching simplifies the logic of this.hasNext() and this.next() + */ + private void fetchNextRecord() { + if (this.leftRecord == null) { throw new NoSuchElementException("No new record to fetch"); } + this.nextRecord = null; + do { + if (this.rightRecord != null) { + DataBox leftJoinValue = this.leftRecord.getValues().get(SNLJOperator.this.getLeftColumnIndex()); + DataBox rightJoinValue = rightRecord.getValues().get(SNLJOperator.this.getRightColumnIndex()); + if (leftJoinValue.equals(rightJoinValue)) { + List leftValues = new ArrayList<>(this.leftRecord.getValues()); + List rightValues = new ArrayList<>(rightRecord.getValues()); + leftValues.addAll(rightValues); + this.nextRecord = new Record(leftValues); + } + this.rightRecord = rightIterator.hasNext() ? rightIterator.next() : null; + } else { + nextLeftRecord(); + resetRightRecord(); + } + } while (!hasNext()); + } + + /** + * Checks if there are more record(s) to yield + * + * @return true if this iterator has another record to yield, otherwise false + */ + @Override + public boolean hasNext() { + return this.nextRecord != null; + } + + /** + * Yields the next record of this iterator. + * + * @return the next Record + * @throws NoSuchElementException if there are no more Records to yield + */ + @Override + public Record next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + + Record nextRecord = this.nextRecord; + try { + this.fetchNextRecord(); + } catch (NoSuchElementException e) { + this.nextRecord = null; + } + return nextRecord; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + +} + diff --git a/src/main/java/edu/berkeley/cs186/database/query/SelectOperator.java b/src/main/java/edu/berkeley/cs186/database/query/SelectOperator.java new file mode 100644 index 0000000..df5a342 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/SelectOperator.java @@ -0,0 +1,183 @@ +package edu.berkeley.cs186.database.query; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.table.MarkerRecord; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.stats.TableStats; + +class SelectOperator extends QueryOperator { + private int columnIndex; + private String columnName; + private PredicateOperator operator; + private DataBox value; + + /** + * Creates a new SelectOperator that pulls from source and only returns tuples for which the + * predicate is satisfied. + * + * @param source the source of this operator + * @param columnName the name of the column to evaluate the predicate on + * @param operator the actual comparator + * @param value the value to compare against + */ + SelectOperator(QueryOperator source, + String columnName, + PredicateOperator operator, + DataBox value) { + super(OperatorType.SELECT, source); + this.operator = operator; + this.value = value; + + this.columnName = this.checkSchemaForColumn(source.getOutputSchema(), columnName); + this.columnIndex = this.getOutputSchema().getFieldNames().indexOf(this.columnName); + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public boolean isSelect() { + return true; + } + + @Override + public Schema computeSchema() { + return this.getSource().getOutputSchema(); + } + + @Override + public String str() { + return "type: " + this.getType() + + "\ncolumn: " + this.columnName + + "\noperator: " + this.operator + + "\nvalue: " + this.value; + } + + /** + * Estimates the table statistics for the result of executing this query operator. + * + * @return estimated TableStats + */ + @Override + public TableStats estimateStats() { + TableStats stats = this.getSource().getStats(); + return stats.copyWithPredicate(this.columnIndex, + this.operator, + this.value); + } + + @Override + public int estimateIOCost() { + return this.getSource().getIOCost(); + } + + @Override + public Iterator iterator() { return new SelectIterator(); } + + /** + * An implementation of Iterator that provides an iterator interface for this operator. + */ + private class SelectIterator implements Iterator { + private Iterator sourceIterator; + private MarkerRecord markerRecord; + private Record nextRecord; + + private SelectIterator() { + this.sourceIterator = SelectOperator.this.getSource().iterator(); + this.markerRecord = MarkerRecord.getMarker(); + this.nextRecord = null; + } + + /** + * Checks if there are more record(s) to yield + * + * @return true if this iterator has another record to yield, otherwise false + */ + @Override + public boolean hasNext() { + if (this.nextRecord != null) { + return true; + } + while (this.sourceIterator.hasNext()) { + Record r = this.sourceIterator.next(); + if (r == this.markerRecord) { + this.nextRecord = r; + return true; + } + switch (SelectOperator.this.operator) { + case EQUALS: + if (r.getValues().get(SelectOperator.this.columnIndex).equals(value)) { + this.nextRecord = r; + return true; + } + break; + case NOT_EQUALS: + if (!r.getValues().get(SelectOperator.this.columnIndex).equals(value)) { + this.nextRecord = r; + return true; + } + break; + case LESS_THAN: + if (r.getValues().get(SelectOperator.this.columnIndex).compareTo(value) < 0) { + this.nextRecord = r; + return true; + } + break; + case LESS_THAN_EQUALS: + if (r.getValues().get(SelectOperator.this.columnIndex).compareTo(value) < 0) { + this.nextRecord = r; + return true; + } else if (r.getValues().get(SelectOperator.this.columnIndex).compareTo(value) == 0) { + this.nextRecord = r; + return true; + } + break; + case GREATER_THAN: + if (r.getValues().get(SelectOperator.this.columnIndex).compareTo(value) > 0) { + this.nextRecord = r; + return true; + } + break; + case GREATER_THAN_EQUALS: + if (r.getValues().get(SelectOperator.this.columnIndex).compareTo(value) > 0) { + this.nextRecord = r; + return true; + } else if (r.getValues().get(SelectOperator.this.columnIndex).compareTo(value) == 0) { + this.nextRecord = r; + return true; + } + break; + default: + break; + } + } + return false; + } + + /** + * Yields the next record of this iterator. + * + * @return the next Record + * @throws NoSuchElementException if there are no more Records to yield + */ + @Override + public Record next() { + if (this.hasNext()) { + Record r = this.nextRecord; + this.nextRecord = null; + return r; + } + throw new NoSuchElementException(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/SequentialScanOperator.java b/src/main/java/edu/berkeley/cs186/database/query/SequentialScanOperator.java new file mode 100644 index 0000000..fcaafbf --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/SequentialScanOperator.java @@ -0,0 +1,92 @@ +package edu.berkeley.cs186.database.query; + +import java.util.Iterator; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.stats.TableStats; + +class SequentialScanOperator extends QueryOperator { + private TransactionContext transaction; + private String tableName; + + /** + * Creates a new SequentialScanOperator that provides an iterator on all tuples in a table. + * + * NOTE: Sequential scans don't take a source operator because they must always be at the bottom + * of the DAG. + * + * @param transaction + * @param tableName + */ + SequentialScanOperator(TransactionContext transaction, + String tableName) { + this(OperatorType.SEQSCAN, transaction, tableName); + } + + protected SequentialScanOperator(OperatorType type, + TransactionContext transaction, + String tableName) { + super(type); + this.transaction = transaction; + this.tableName = tableName; + this.setOutputSchema(this.computeSchema()); + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + public String getTableName() { + return this.tableName; + } + + @Override + public boolean isSequentialScan() { + return true; + } + + @Override + public Iterator iterator() { + return this.transaction.getRecordIterator(tableName); + } + + @Override + public Schema computeSchema() { + try { + return this.transaction.getFullyQualifiedSchema(this.tableName); + } catch (DatabaseException de) { + throw new QueryPlanException(de); + } + } + + @Override + public String str() { + return "type: " + this.getType() + + "\ntable: " + this.tableName; + } + + /** + * Estimates the table statistics for the result of executing this query operator. + * + * @return estimated TableStats + */ + @Override + public TableStats estimateStats() { + try { + return this.transaction.getStats(this.tableName); + } catch (DatabaseException de) { + throw new QueryPlanException(de); + } + } + + @Override + public int estimateIOCost() { + try { + return this.transaction.getNumDataPages(this.tableName); + } catch (DatabaseException de) { + throw new QueryPlanException(de); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/SortMergeOperator.java b/src/main/java/edu/berkeley/cs186/database/query/SortMergeOperator.java new file mode 100644 index 0000000..bda66c5 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/SortMergeOperator.java @@ -0,0 +1,108 @@ +package edu.berkeley.cs186.database.query; + +import java.util.*; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.table.Record; + +class SortMergeOperator extends JoinOperator { + SortMergeOperator(QueryOperator leftSource, + QueryOperator rightSource, + String leftColumnName, + String rightColumnName, + TransactionContext transaction) { + super(leftSource, rightSource, leftColumnName, rightColumnName, transaction, JoinType.SORTMERGE); + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public Iterator iterator() { + return new SortMergeIterator(); + } + + @Override + public int estimateIOCost() { + //does nothing + return 0; + } + + /** + * An implementation of Iterator that provides an iterator interface for this operator. + * + * Before proceeding, you should read and understand SNLJOperator.java + * You can find it in the same directory as this file. + * + * Word of advice: try to decompose the problem into distinguishable sub-problems. + * This means you'll probably want to add more methods than those given (Once again, + * SNLJOperator.java might be a useful reference). + * + */ + private class SortMergeIterator extends JoinIterator { + /** + * Some member variables are provided for guidance, but there are many possible solutions. + * You should implement the solution that's best for you, using any member variables you need. + * You're free to use these member variables, but you're not obligated to. + */ + private BacktrackingIterator leftIterator; + private BacktrackingIterator rightIterator; + private Record leftRecord; + private Record nextRecord; + private Record rightRecord; + private boolean marked; + + private SortMergeIterator() { + super(); + // TODO(hw3_part1): implement + } + + /** + * Checks if there are more record(s) to yield + * + * @return true if this iterator has another record to yield, otherwise false + */ + @Override + public boolean hasNext() { + // TODO(hw3_part1): implement + + return false; + } + + /** + * Yields the next record of this iterator. + * + * @return the next Record + * @throws NoSuchElementException if there are no more Records to yield + */ + @Override + public Record next() { + // TODO(hw3_part1): implement + + throw new NoSuchElementException(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + private class LeftRecordComparator implements Comparator { + @Override + public int compare(Record o1, Record o2) { + return o1.getValues().get(SortMergeOperator.this.getLeftColumnIndex()).compareTo( + o2.getValues().get(SortMergeOperator.this.getLeftColumnIndex())); + } + } + + private class RightRecordComparator implements Comparator { + @Override + public int compare(Record o1, Record o2) { + return o1.getValues().get(SortMergeOperator.this.getRightColumnIndex()).compareTo( + o2.getValues().get(SortMergeOperator.this.getRightColumnIndex())); + } + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/query/SortOperator.java b/src/main/java/edu/berkeley/cs186/database/query/SortOperator.java new file mode 100644 index 0000000..b06267a --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/query/SortOperator.java @@ -0,0 +1,210 @@ +package edu.berkeley.cs186.database.query; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.memory.Page; + +import java.util.*; + +public class SortOperator { + private TransactionContext transaction; + private String tableName; + private Comparator comparator; + private Schema operatorSchema; + private int numBuffers; + private String sortedTableName = null; + + public SortOperator(TransactionContext transaction, String tableName, + Comparator comparator) { + this.transaction = transaction; + this.tableName = tableName; + this.comparator = comparator; + this.operatorSchema = this.computeSchema(); + this.numBuffers = this.transaction.getWorkMemSize(); + } + + private Schema computeSchema() { + try { + return this.transaction.getFullyQualifiedSchema(this.tableName); + } catch (DatabaseException de) { + throw new QueryPlanException(de); + } + } + + /** + * Interface for a run. Also see createRun/createRunFromIterator. + */ + public interface Run extends Iterable { + /** + * Add a record to the run. + * @param values set of values of the record to add to run + */ + void addRecord(List values); + + /** + * Add a list of records to the run. + * @param records records to add to the run + */ + void addRecords(List records); + + @Override + Iterator iterator(); + + /** + * Table name of table backing the run. + * @return table name + */ + String tableName(); + } + + /** + * Returns a NEW run that is the sorted version of the input run. + * Can do an in memory sort over all the records in this run + * using one of Java's built-in sorting methods. + * Note: Don't worry about modifying the original run. + * Returning a new run would bring one extra page in memory beyond the + * size of the buffer, but it is done this way for ease. + */ + public Run sortRun(Run run) { + // TODO(hw3_part1): implement + + return null; + } + + /** + * Given a list of sorted runs, returns a new run that is the result + * of merging the input runs. You should use a Priority Queue (java.util.PriorityQueue) + * to determine which record should be should be added to the output run next. + * It is recommended that your Priority Queue hold Pair objects + * where a Pair (r, i) is the Record r with the smallest value you are + * sorting on currently unmerged from run i. + */ + public Run mergeSortedRuns(List runs) { + // TODO(hw3_part1): implement + + return null; + } + + /** + * Given a list of N sorted runs, returns a list of + * sorted runs that is the result of merging (numBuffers - 1) + * of the input runs at a time. + */ + public List mergePass(List runs) { + // TODO(hw3_part1): implement + + return Collections.emptyList(); + } + + /** + * Does an external merge sort on the table with name tableName + * using numBuffers. + * Returns the name of the table that backs the final run. + */ + public String sort() { + // TODO(hw3_part1): implement + + return this.tableName; // TODO(hw3_part1): replace this! + } + + public Iterator iterator() { + if (sortedTableName == null) { + sortedTableName = sort(); + } + return this.transaction.getRecordIterator(sortedTableName); + } + + /** + * Creates a new run for intermediate steps of sorting. The created + * run supports adding records. + * @return a new, empty run + */ + Run createRun() { + return new IntermediateRun(); + } + + /** + * Creates a run given a backtracking iterator of records. Record adding + * is not supported, but creating this run will not incur any I/Os aside + * from any I/Os incurred while reading from the given iterator. + * @param records iterator of records + * @return run backed by the iterator of records + */ + Run createRunFromIterator(BacktrackingIterator records) { + return new InputDataRun(records); + } + + private class IntermediateRun implements Run { + String tempTableName; + + IntermediateRun() { + this.tempTableName = SortOperator.this.transaction.createTempTable( + SortOperator.this.operatorSchema); + } + + @Override + public void addRecord(List values) { + SortOperator.this.transaction.addRecord(this.tempTableName, values); + } + + @Override + public void addRecords(List records) { + for (Record r : records) { + this.addRecord(r.getValues()); + } + } + + @Override + public Iterator iterator() { + return SortOperator.this.transaction.getRecordIterator(this.tempTableName); + } + + @Override + public String tableName() { + return this.tempTableName; + } + } + + private static class InputDataRun implements Run { + BacktrackingIterator iterator; + + InputDataRun(BacktrackingIterator iterator) { + this.iterator = iterator; + this.iterator.markPrev(); + } + + @Override + public void addRecord(List values) { + throw new UnsupportedOperationException("cannot add record to input data run"); + } + + @Override + public void addRecords(List records) { + throw new UnsupportedOperationException("cannot add records to input data run"); + } + + @Override + public Iterator iterator() { + iterator.reset(); + return iterator; + } + + @Override + public String tableName() { + throw new UnsupportedOperationException("cannot get table name of input data run"); + } + } + + private class RecordPairComparator implements Comparator> { + @Override + public int compare(Pair o1, Pair o2) { + return SortOperator.this.comparator.compare(o1.getFirst(), o2.getFirst()); + } + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/ARIESRecoveryManager.java b/src/main/java/edu/berkeley/cs186/database/recovery/ARIESRecoveryManager.java new file mode 100644 index 0000000..13d3fe8 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/ARIESRecoveryManager.java @@ -0,0 +1,620 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.concurrency.LockType; +import edu.berkeley.cs186.database.concurrency.LockUtil; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Implementation of ARIES. + */ +public class ARIESRecoveryManager implements RecoveryManager { + // Lock context of the entire database. + private LockContext dbContext; + // Disk space manager. + DiskSpaceManager diskSpaceManager; + // Buffer manager. + BufferManager bufferManager; + + // Function to create a new transaction for recovery with a given transaction number. + private Function newTransaction; + // Function to update the transaction counter. + protected Consumer updateTransactionCounter; + // Function to get the transaction counter. + protected Supplier getTransactionCounter; + + // Log manager + LogManager logManager; + // Dirty page table (page number -> recLSN). + Map dirtyPageTable = new ConcurrentHashMap<>(); + // Transaction table (transaction number -> entry). + Map transactionTable = new ConcurrentHashMap<>(); + + // List of lock requests made during recovery. This is only populated when locking is disabled. + List lockRequests; + + public ARIESRecoveryManager(LockContext dbContext, Function newTransaction, + Consumer updateTransactionCounter, Supplier getTransactionCounter) { + this(dbContext, newTransaction, updateTransactionCounter, getTransactionCounter, false); + } + + ARIESRecoveryManager(LockContext dbContext, Function newTransaction, + Consumer updateTransactionCounter, Supplier getTransactionCounter, + boolean disableLocking) { + this.dbContext = dbContext; + this.newTransaction = newTransaction; + this.updateTransactionCounter = updateTransactionCounter; + this.getTransactionCounter = getTransactionCounter; + this.lockRequests = disableLocking ? new ArrayList<>() : null; + } + + /** + * Initializes the log; only called the first time the database is set up. + * + * The master record should be added to the log, and a checkpoint should be taken. + */ + @Override + public void initialize() { + this.logManager.appendToLog(new MasterLogRecord(0)); + this.checkpoint(); + } + + /** + * Sets the buffer/disk managers. This is not part of the constructor because of the cyclic dependency + * between the buffer manager and recovery manager (the buffer manager must interface with the + * recovery manager to block page evictions until the log has been flushed, but the recovery + * manager needs to interface with the buffer manager to write the log and redo changes). + * @param diskSpaceManager disk space manager + * @param bufferManager buffer manager + */ + @Override + public void setManagers(DiskSpaceManager diskSpaceManager, BufferManager bufferManager) { + this.diskSpaceManager = diskSpaceManager; + this.bufferManager = bufferManager; + this.logManager = new LogManagerImpl(bufferManager); + } + + // Forward Processing //////////////////////////////////////////////////////////////////// + + /** + * Called when a new transaction is started. + * + * The transaction should be added to the transaction table. + * + * @param transaction new transaction + */ + @Override + public synchronized void startTransaction(Transaction transaction) { + this.transactionTable.put(transaction.getTransNum(), new TransactionTableEntry(transaction)); + } + + /** + * Called when a transaction is about to start committing. + * + * A commit record should be emitted, the log should be flushed, + * and the transaction table and the transaction status should be updated. + * + * @param transNum transaction being committed + * @return LSN of the commit record + */ + @Override + public long commit(long transNum) { + // TODO(hw5): implement + return -1L; + } + + /** + * Called when a transaction is set to be aborted. + * + * An abort record should be emitted, and the transaction table and transaction + * status should be updated. No CLRs should be emitted. + * + * @param transNum transaction being aborted + * @return LSN of the abort record + */ + @Override + public long abort(long transNum) { + // TODO(hw5): implement + return -1L; + } + + /** + * Called when a transaction is cleaning up; this should roll back + * changes if the transaction is aborting. + * + * Any changes that need to be undone should be undone, the transaction should + * be removed from the transaction table, the end record should be emitted, + * and the transaction status should be updated. + * + * @param transNum transaction to end + * @return LSN of the end record + */ + @Override + public long end(long transNum) { + // TODO(hw5): implement + return -1L; + } + + /** + * Called before a page is flushed from the buffer cache. This + * method is never called on a log page. + * + * The log should be as far as necessary. + * + * @param pageLSN pageLSN of page about to be flushed + */ + @Override + public void pageFlushHook(long pageLSN) { + logManager.flushToLSN(pageLSN); + } + + /** + * Called when a page has been updated on disk. + * + * As the page is no longer dirty, it should be removed from the + * dirty page table. + * + * @param pageNum page number of page updated on disk + */ + @Override + public void diskIOHook(long pageNum) { + dirtyPageTable.remove(pageNum); + } + + /** + * Called when a write to a page happens. + * + * This method is never called on a log page. Arguments to the before and after params + * are guaranteed to be the same length. + * + * The appropriate log record should be emitted; if the number of bytes written is + * too large (larger than BufferManager.EFFECTIVE_PAGE_SIZE / 2), then two records + * should be written instead: an undo-only record followed by a redo-only record. + * + * Both the transaction table and dirty page table should be updated accordingly. + * + * @param transNum transaction performing the write + * @param pageNum page number of page being written + * @param pageOffset offset into page where write begins + * @param before bytes starting at pageOffset before the write + * @param after bytes starting at pageOffset after the write + * @return LSN of last record written to log + */ + @Override + public long logPageWrite(long transNum, long pageNum, short pageOffset, byte[] before, + byte[] after) { + assert (before.length == after.length); + + // TODO(hw5): implement + return -1L; + } + + /** + * Called when a new partition is allocated. A log flush is necessary, + * since changes are visible on disk immediately after this returns. + * + * This method should return -1 if the partition is the log partition. + * + * The appropriate log record should be emitted, and the log flushed. + * The transaction table should be updated accordingly. + * + * @param transNum transaction requesting the allocation + * @param partNum partition number of the new partition + * @return LSN of record or -1 if log partition + */ + @Override + public long logAllocPart(long transNum, int partNum) { + // Ignore if part of the log. + if (partNum == 0) { + return -1L; + } + + TransactionTableEntry transactionEntry = transactionTable.get(transNum); + assert (transactionEntry != null); + + long prevLSN = transactionEntry.lastLSN; + LogRecord record = new AllocPartLogRecord(transNum, partNum, prevLSN); + long LSN = logManager.appendToLog(record); + // Update lastLSN + transactionEntry.lastLSN = LSN; + // Flush log + logManager.flushToLSN(LSN); + return LSN; + } + + /** + * Called when a partition is freed. A log flush is necessary, + * since changes are visible on disk immediately after this returns. + * + * This method should return -1 if the partition is the log partition. + * + * The appropriate log record should be emitted, and the log flushed. + * The transaction table should be updated accordingly. + * + * @param transNum transaction requesting the partition be freed + * @param partNum partition number of the partition being freed + * @return LSN of record or -1 if log partition + */ + @Override + public long logFreePart(long transNum, int partNum) { + // Ignore if part of the log. + if (partNum == 0) { + return -1L; + } + + TransactionTableEntry transactionEntry = transactionTable.get(transNum); + assert (transactionEntry != null); + + long prevLSN = transactionEntry.lastLSN; + LogRecord record = new FreePartLogRecord(transNum, partNum, prevLSN); + long LSN = logManager.appendToLog(record); + // Update lastLSN + transactionEntry.lastLSN = LSN; + // Flush log + logManager.flushToLSN(LSN); + return LSN; + } + + /** + * Called when a new page is allocated. A log flush is necessary, + * since changes are visible on disk immediately after this returns. + * + * This method should return -1 if the page is in the log partition. + * + * The appropriate log record should be emitted, and the log flushed. + * The transaction table should be updated accordingly. + * + * @param transNum transaction requesting the allocation + * @param pageNum page number of the new page + * @return LSN of record or -1 if log partition + */ + @Override + public long logAllocPage(long transNum, long pageNum) { + // Ignore if part of the log. + if (DiskSpaceManager.getPartNum(pageNum) == 0) { + return -1L; + } + + TransactionTableEntry transactionEntry = transactionTable.get(transNum); + assert (transactionEntry != null); + + long prevLSN = transactionEntry.lastLSN; + LogRecord record = new AllocPageLogRecord(transNum, pageNum, prevLSN); + long LSN = logManager.appendToLog(record); + // Update lastLSN, touchedPages + transactionEntry.lastLSN = LSN; + transactionEntry.touchedPages.add(pageNum); + // Flush log + logManager.flushToLSN(LSN); + return LSN; + } + + /** + * Called when a page is freed. A log flush is necessary, + * since changes are visible on disk immediately after this returns. + * + * This method should return -1 if the page is in the log partition. + * + * The appropriate log record should be emitted, and the log flushed. + * The transaction table should be updated accordingly. + * + * @param transNum transaction requesting the page be freed + * @param pageNum page number of the page being freed + * @return LSN of record or -1 if log partition + */ + @Override + public long logFreePage(long transNum, long pageNum) { + // Ignore if part of the log. + if (DiskSpaceManager.getPartNum(pageNum) == 0) { + return -1L; + } + + TransactionTableEntry transactionEntry = transactionTable.get(transNum); + assert (transactionEntry != null); + + long prevLSN = transactionEntry.lastLSN; + LogRecord record = new FreePageLogRecord(transNum, pageNum, prevLSN); + long LSN = logManager.appendToLog(record); + // Update lastLSN, touchedPages + transactionEntry.lastLSN = LSN; + transactionEntry.touchedPages.add(pageNum); + dirtyPageTable.remove(pageNum); + // Flush log + logManager.flushToLSN(LSN); + return LSN; + } + + /** + * Creates a savepoint for a transaction. Creating a savepoint with + * the same name as an existing savepoint for the transaction should + * delete the old savepoint. + * + * The appropriate LSN should be recorded so that a partial rollback + * is possible later. + * + * @param transNum transaction to make savepoint for + * @param name name of savepoint + */ + @Override + public void savepoint(long transNum, String name) { + TransactionTableEntry transactionEntry = transactionTable.get(transNum); + assert (transactionEntry != null); + + transactionEntry.addSavepoint(name); + } + + /** + * Releases (deletes) a savepoint for a transaction. + * @param transNum transaction to delete savepoint for + * @param name name of savepoint + */ + @Override + public void releaseSavepoint(long transNum, String name) { + TransactionTableEntry transactionEntry = transactionTable.get(transNum); + assert (transactionEntry != null); + + transactionEntry.deleteSavepoint(name); + } + + /** + * Rolls back transaction to a savepoint. + * + * All changes done by the transaction since the savepoint should be undone, + * in reverse order, with the appropriate CLRs written to log. The transaction + * status should remain unchanged. + * + * @param transNum transaction to partially rollback + * @param name name of savepoint + */ + @Override + public void rollbackToSavepoint(long transNum, String name) { + TransactionTableEntry transactionEntry = transactionTable.get(transNum); + assert (transactionEntry != null); + + // All of the transaction's changes strictly after the record at LSN should be undone. + long LSN = transactionEntry.getSavepoint(name); + + // TODO(hw5): implement + return; + } + + /** + * Create a checkpoint. + * + * First, a begin checkpoint record should be written. + * + * Then, end checkpoint records should be filled up as much as possible, + * using recLSNs from the DPT, then status/lastLSNs from the transactions table, + * and then finally, touchedPages from the transactions table, and written + * when full (or when done). + * + * Finally, the master record should be rewritten with the LSN of the + * begin checkpoint record. + */ + @Override + public void checkpoint() { + // Create begin checkpoint log record and write to log + LogRecord beginRecord = new BeginCheckpointLogRecord(getTransactionCounter.get()); + long beginLSN = logManager.appendToLog(beginRecord); + + Map dpt = new HashMap<>(); + Map> txnTable = new HashMap<>(); + Map> touchedPages = new HashMap<>(); + int numTouchedPages = 0; + + // TODO(hw5): generate end checkpoint record(s) for DPT and transaction table + + for (Map.Entry entry : transactionTable.entrySet()) { + long transNum = entry.getKey(); + for (long pageNum : entry.getValue().touchedPages) { + boolean fitsAfterAdd; + if (!touchedPages.containsKey(transNum)) { + fitsAfterAdd = EndCheckpointLogRecord.fitsInOneRecord( + dpt.size(), txnTable.size(), touchedPages.size() + 1, numTouchedPages + 1); + } else { + fitsAfterAdd = EndCheckpointLogRecord.fitsInOneRecord( + dpt.size(), txnTable.size(), touchedPages.size(), numTouchedPages + 1); + } + + if (!fitsAfterAdd) { + LogRecord endRecord = new EndCheckpointLogRecord(dpt, txnTable, touchedPages); + logManager.appendToLog(endRecord); + + dpt.clear(); + txnTable.clear(); + touchedPages.clear(); + numTouchedPages = 0; + } + + touchedPages.computeIfAbsent(transNum, t -> new ArrayList<>()); + touchedPages.get(transNum).add(pageNum); + ++numTouchedPages; + } + } + + // Last end checkpoint record + LogRecord endRecord = new EndCheckpointLogRecord(dpt, txnTable, touchedPages); + logManager.appendToLog(endRecord); + + // Update master record + MasterLogRecord masterRecord = new MasterLogRecord(beginLSN); + logManager.rewriteMasterRecord(masterRecord); + } + + // TODO(hw5): add any helper methods needed + + @Override + public void close() { + this.checkpoint(); + this.logManager.close(); + } + + // Restart Recovery ////////////////////////////////////////////////////////////////////// + + /** + * Called whenever the database starts up, and performs restart recovery. Recovery is + * complete when the Runnable returned is run to termination. New transactions may be + * started once this method returns. + * + * This should perform the three phases of recovery, and also clean the dirty page + * table of non-dirty pages (pages that aren't dirty in the buffer manager) between + * redo and undo, and perform a checkpoint after undo. + * + * This method should return right before undo is performed. + * + * @return Runnable to run to finish restart recovery + */ + @Override + public Runnable restart() { + // TODO(hw5): implement + return () -> {}; + } + + /** + * This method performs the analysis pass of restart recovery. + * + * First, the master record should be read (LSN 0). The master record contains + * one piece of information: the LSN of the last successful checkpoint. + * + * We then begin scanning log records, starting at the begin checkpoint record. + * + * If the log record is for a transaction operation: + * - update the transaction table + * - if it's page-related (as opposed to partition-related), + * - add to touchedPages + * - acquire X lock + * - update DPT (alloc/free/undoalloc/undofree always flushes changes to disk) + * + * If the log record is for a change in transaction status: + * - clean up transaction (Transaction#cleanup) if END_TRANSACTION + * - update transaction status to COMMITTING/RECOVERY_ABORTING/COMPLETE + * - update the transaction table + * + * If the log record is a begin_checkpoint record: + * - Update the transaction counter + * + * If the log record is an end_checkpoint record: + * - Copy all entries of checkpoint DPT (replace existing entries if any) + * - Update lastLSN to be the larger of the existing entry's (if any) and the checkpoint's; + * add to transaction table if not already present. + * - Add page numbers from checkpoint's touchedPages to the touchedPages sets in the + * transaction table if the transaction has not finished yet, and acquire X locks. + * + * Then, cleanup and end transactions that are in the COMMITING state, and + * move all transactions in the RUNNING state to RECOVERY_ABORTING. + */ + void restartAnalysis() { + // Read master record + LogRecord record = logManager.fetchLogRecord(0L); + assert (record != null); + // Type casting + assert (record.getType() == LogType.MASTER); + MasterLogRecord masterRecord = (MasterLogRecord) record; + // Get start checkpoint LSN + long LSN = masterRecord.lastCheckpointLSN; + + // TODO(hw5): implement + return; + } + + /** + * This method performs the redo pass of restart recovery. + * + * First, determine the starting point for REDO from the DPT. + * + * Then, scanning from the starting point, if the record is redoable and + * - about a page (Update/Alloc/Free/Undo..Page) in the DPT with LSN >= recLSN, + * the page is fetched from disk and the pageLSN is checked, and the record is redone. + * - about a partition (Alloc/Free/Undo..Part), redo it. + */ + void restartRedo() { + // TODO(hw5): implement + return; + } + + /** + * This method performs the redo pass of restart recovery. + + * First, a priority queue is created sorted on lastLSN of all aborting transactions. + * + * Then, always working on the largest LSN in the priority queue until we are done, + * - if the record is undoable, undo it, emit the appropriate CLR, and update tables accordingly; + * - replace the entry in the set should be replaced with a new one, using the undoNextLSN + * (or prevLSN if none) of the record; and + * - if the new LSN is 0, end the transaction and remove it from the queue and transaction table. + */ + void restartUndo() { + // TODO(hw5): implement + return; + } + + // TODO(hw5): add any helper methods needed + + // Helpers /////////////////////////////////////////////////////////////////////////////// + + /** + * Returns the lock context for a given page number. + * @param pageNum page number to get lock context for + * @return lock context of the page + */ + private LockContext getPageLockContext(long pageNum) { + int partNum = DiskSpaceManager.getPartNum(pageNum); + return this.dbContext.childContext(partNum).childContext(pageNum); + } + + /** + * Locks the given lock context with the specified lock type under the specified transaction, + * acquiring locks on ancestors as needed. + * @param transaction transaction to request lock for + * @param lockContext lock context to lock + * @param lockType type of lock to request + */ + private void acquireTransactionLock(Transaction transaction, LockContext lockContext, + LockType lockType) { + acquireTransactionLock(transaction.getTransactionContext(), lockContext, lockType); + } + + /** + * Locks the given lock context with the specified lock type under the specified transaction, + * acquiring locks on ancestors as needed. + * @param transactionContext transaction context to request lock for + * @param lockContext lock context to lock + * @param lockType type of lock to request + */ + private void acquireTransactionLock(TransactionContext transactionContext, + LockContext lockContext, LockType lockType) { + TransactionContext.setTransaction(transactionContext); + try { + if (lockRequests == null) { + LockUtil.ensureSufficientLockHeld(lockContext, lockType); + } else { + lockRequests.add("request " + transactionContext.getTransNum() + " " + lockType + "(" + + lockContext.getResourceName() + ")"); + } + } finally { + TransactionContext.unsetTransaction(); + } + } + + /** + * Comparator for Pair comparing only on the first element (type A), in reverse order. + */ + private static class PairFirstReverseComparator, B> implements + Comparator> { + @Override + public int compare(Pair p0, Pair p1) { + return p1.getFirst().compareTo(p0.getFirst()); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/AbortTransactionLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/AbortTransactionLogRecord.java new file mode 100644 index 0000000..57f6a5d --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/AbortTransactionLogRecord.java @@ -0,0 +1,68 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import java.util.Objects; +import java.util.Optional; + +class AbortTransactionLogRecord extends LogRecord { + private long transNum; + private long prevLSN; + + AbortTransactionLogRecord(long transNum, long prevLSN) { + super(LogType.ABORT_TRANSACTION); + this.transNum = transNum; + this.prevLSN = prevLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(prevLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long prevLSN = buf.getLong(); + return Optional.of(new AbortTransactionLogRecord(transNum, prevLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + AbortTransactionLogRecord that = (AbortTransactionLogRecord) o; + return transNum == that.transNum && + prevLSN == that.prevLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, prevLSN); + } + + @Override + public String toString() { + return "AbortTransactionLogRecord{" + + "transNum=" + transNum + + ", prevLSN=" + prevLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/AllocPageLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/AllocPageLogRecord.java new file mode 100644 index 0000000..7f38af3 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/AllocPageLogRecord.java @@ -0,0 +1,111 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; + +import java.util.Objects; +import java.util.Optional; + +/** + * A log entry that records the allocation of a page + */ +class AllocPageLogRecord extends LogRecord { + private long transNum; + private long pageNum; + private long prevLSN; + + AllocPageLogRecord(long transNum, long pageNum, long prevLSN) { + super(LogType.ALLOC_PAGE); + this.transNum = transNum; + this.pageNum = pageNum; + this.prevLSN = prevLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPageNum() { + return Optional.of(pageNum); + } + + @Override + public boolean isUndoable() { + return true; + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public Pair undo(long lastLSN) { + return new Pair<>(new UndoAllocPageLogRecord(transNum, pageNum, lastLSN, prevLSN), true); + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + try { + dsm.allocPage(pageNum); + } catch (IllegalStateException e) { + /* do nothing - page already exists */ + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(pageNum) + .putLong(prevLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long pageNum = buf.getLong(); + long prevLSN = buf.getLong(); + return Optional.of(new AllocPageLogRecord(transNum, pageNum, prevLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + AllocPageLogRecord that = (AllocPageLogRecord) o; + return transNum == that.transNum && + pageNum == that.pageNum && + prevLSN == that.prevLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, pageNum, prevLSN); + } + + @Override + public String toString() { + return "AllocPageLogRecord{" + + "transNum=" + transNum + + ", pageNum=" + pageNum + + ", prevLSN=" + prevLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/AllocPartLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/AllocPartLogRecord.java new file mode 100644 index 0000000..ea298b2 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/AllocPartLogRecord.java @@ -0,0 +1,111 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; + +import java.util.Objects; +import java.util.Optional; + +/** + * A log entry that records the allocation of a partition + */ +class AllocPartLogRecord extends LogRecord { + private long transNum; + private int partNum; + private long prevLSN; + + AllocPartLogRecord(long transNum, int partNum, long prevLSN) { + super(LogType.ALLOC_PART); + this.transNum = transNum; + this.partNum = partNum; + this.prevLSN = prevLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPartNum() { + return Optional.of(partNum); + } + + @Override + public boolean isUndoable() { + return true; + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public Pair undo(long lastLSN) { + return new Pair<>(new UndoAllocPartLogRecord(transNum, partNum, lastLSN, prevLSN), true); + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + try { + dsm.allocPart(partNum); + } catch (IllegalStateException e) { + /* do nothing - partition already exists */ + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Integer.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putInt(partNum) + .putLong(prevLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + int partNum = buf.getInt(); + long prevLSN = buf.getLong(); + return Optional.of(new AllocPartLogRecord(transNum, partNum, prevLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + AllocPartLogRecord that = (AllocPartLogRecord) o; + return transNum == that.transNum && + partNum == that.partNum && + prevLSN == that.prevLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, partNum, prevLSN); + } + + @Override + public String toString() { + return "AllocPartLogRecord{" + + "transNum=" + transNum + + ", partNum=" + partNum + + ", prevLSN=" + prevLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/BeginCheckpointLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/BeginCheckpointLogRecord.java new file mode 100644 index 0000000..fcf9728 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/BeginCheckpointLogRecord.java @@ -0,0 +1,54 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import java.util.Objects; +import java.util.Optional; + +class BeginCheckpointLogRecord extends LogRecord { + long maxTransNum; + + BeginCheckpointLogRecord(long maxTransNum) { + super(LogType.BEGIN_CHECKPOINT); + this.maxTransNum = maxTransNum; + } + + @Override + public Optional getMaxTransactionNum() { + return Optional.of(maxTransNum); + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[9]; + ByteBuffer.wrap(b).put((byte) getType().getValue()).putLong(maxTransNum); + return b; + } + + public static Optional fromBytes(Buffer buf) { + return Optional.of(new BeginCheckpointLogRecord(buf.getLong())); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + BeginCheckpointLogRecord that = (BeginCheckpointLogRecord) o; + return maxTransNum == that.maxTransNum; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), maxTransNum); + } + + @Override + public String toString() { + return "BeginCheckpointLogRecord{" + + "maxTransNum=" + maxTransNum + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/CommitTransactionLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/CommitTransactionLogRecord.java new file mode 100644 index 0000000..e614cf4 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/CommitTransactionLogRecord.java @@ -0,0 +1,68 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import java.util.Objects; +import java.util.Optional; + +class CommitTransactionLogRecord extends LogRecord { + private long transNum; + private long prevLSN; + + CommitTransactionLogRecord(long transNum, long prevLSN) { + super(LogType.COMMIT_TRANSACTION); + this.transNum = transNum; + this.prevLSN = prevLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(prevLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long prevLSN = buf.getLong(); + return Optional.of(new CommitTransactionLogRecord(transNum, prevLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + CommitTransactionLogRecord that = (CommitTransactionLogRecord) o; + return transNum == that.transNum && + prevLSN == that.prevLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, prevLSN); + } + + @Override + public String toString() { + return "CommitTransactionLogRecord{" + + "transNum=" + transNum + + ", prevLSN=" + prevLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/DummyRecoveryManager.java b/src/main/java/edu/berkeley/cs186/database/recovery/DummyRecoveryManager.java new file mode 100644 index 0000000..3281f67 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/DummyRecoveryManager.java @@ -0,0 +1,101 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; + +import java.util.HashMap; +import java.util.Map; + +public class DummyRecoveryManager implements RecoveryManager { + private Map runningTransactions = new HashMap<>(); + + @Override + public void initialize() {} + + @Override + public void setManagers(DiskSpaceManager diskSpaceManager, BufferManager bufferManager) {} + + @Override + public void startTransaction(Transaction transaction) { + runningTransactions.put(transaction.getTransNum(), transaction); + } + + @Override + public long commit(long transNum) { + runningTransactions.get(transNum).setStatus(Transaction.Status.COMMITTING); + return 0L; + } + + @Override + public long abort(long transNum) { + throw new UnsupportedOperationException("hw5 must be implemented to use abort"); + } + + @Override + public long end(long transNum) { + runningTransactions.get(transNum).setStatus(Transaction.Status.COMPLETE); + runningTransactions.remove(transNum); + return 0L; + } + + @Override + public void pageFlushHook(long pageLSN) {} + + @Override + public void diskIOHook(long pageNum) {} + + @Override + public long logPageWrite(long transNum, long pageNum, short pageOffset, byte[] before, + byte[] after) { + return 0L; + } + + @Override + public long logAllocPart(long transNum, int partNum) { + return 0L; + } + + @Override + public long logFreePart(long transNum, int partNum) { + return 0L; + } + + @Override + public long logAllocPage(long transNum, long pageNum) { + return 0L; + } + + @Override + public long logFreePage(long transNum, long pageNum) { + return 0L; + } + + @Override + public void savepoint(long transNum, String name) { + throw new UnsupportedOperationException("hw5 must be implemented to use savepoints"); + } + + @Override + public void releaseSavepoint(long transNum, String name) { + throw new UnsupportedOperationException("hw5 must be implemented to use savepoints"); + } + + @Override + public void rollbackToSavepoint(long transNum, String name) { + throw new UnsupportedOperationException("hw5 must be implemented to use savepoints"); + } + + @Override + public void checkpoint() { + throw new UnsupportedOperationException("hw5 must be implemented to use checkpoints"); + } + + @Override + public Runnable restart() { + return () -> {}; + } + + @Override + public void close() {} +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/EndCheckpointLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/EndCheckpointLogRecord.java new file mode 100644 index 0000000..05c261b --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/EndCheckpointLogRecord.java @@ -0,0 +1,149 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.io.DiskSpaceManager; + +import java.util.*; + +class EndCheckpointLogRecord extends LogRecord { + private Map dirtyPageTable; + private Map> transactionTable; + private Map> touchedPages; + private int numTouchedPages; + + public EndCheckpointLogRecord(Map dirtyPageTable, + Map> transactionTable, + Map> touchedPages) { + super(LogType.END_CHECKPOINT); + this.dirtyPageTable = new HashMap<>(dirtyPageTable); + this.transactionTable = new HashMap<>(transactionTable); + this.touchedPages = new HashMap<>(touchedPages); + this.numTouchedPages = 0; + for (List pages : touchedPages.values()) { + this.numTouchedPages += pages.size(); + } + } + + @Override + public Map getDirtyPageTable() { + return dirtyPageTable; + } + + @Override + public Map> getTransactionTable() { + return transactionTable; + } + + @Override + public Map> getTransactionTouchedPages() { + return touchedPages; + } + + @Override + public byte[] toBytes() { + int recordSize = getRecordSize(dirtyPageTable.size(), transactionTable.size(), touchedPages.size(), + numTouchedPages); + byte[] b = new byte[recordSize]; + Buffer buf = ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putShort((short) dirtyPageTable.size()) + .putShort((short) transactionTable.size()) + .putShort((short) touchedPages.size()); + for (Map.Entry entry : dirtyPageTable.entrySet()) { + buf.putLong(entry.getKey()).putLong(entry.getValue()); + } + for (Map.Entry> entry : transactionTable.entrySet()) { + buf.putLong(entry.getKey()) + .put((byte) entry.getValue().getFirst().ordinal()) + .putLong(entry.getValue().getSecond()); + } + for (Map.Entry> entry : touchedPages.entrySet()) { + buf.putLong(entry.getKey()) + .putShort((short) entry.getValue().size()); + for (Long pageNum : entry.getValue()) { + buf.putLong(pageNum); + } + } + return b; + } + + /** + * @return size of the record in bytes + */ + public static int getRecordSize(int numDPTRecords, int numTxnTableRecords, + int touchedPagesMapSize, int numTouchedPages) { + // DPT: long -> long (16 bytes) + // xact: long -> (byte, long) (17 bytes) + // touched pages: long -> (short (size) + longs) (10 bytes + 8 bytes per page) + return 7 + 16 * numDPTRecords + 17 * numTxnTableRecords + 10 * touchedPagesMapSize + 8 * + numTouchedPages; + } + + /** + * @return boolean indicating whether information for + * the log record can fit in one record on a page + */ + public static boolean fitsInOneRecord(int numDPTRecords, int numTxnTableRecords, + int touchedPagesMapSize, int numTouchedPages) { + int recordSize = getRecordSize(numDPTRecords, numTxnTableRecords, touchedPagesMapSize, + numTouchedPages); + return recordSize <= DiskSpaceManager.PAGE_SIZE; + } + + public static Optional fromBytes(Buffer buf) { + short dptSize = buf.getShort(); + short xactSize = buf.getShort(); + short tpSize = buf.getShort(); + Map dirtyPageTable = new HashMap<>(); + Map> transactionTable = new HashMap<>(); + Map> touchedPages = new HashMap<>(); + for (short i = 0; i < dptSize; ++i) { + dirtyPageTable.put(buf.getLong(), buf.getLong()); + } + for (short i = 0; i < xactSize; ++i) { + long transNum = buf.getLong(); + Transaction.Status status = Transaction.Status.fromInt(buf.get()); + long lastLSN = buf.getLong(); + transactionTable.put(transNum, new Pair<>(status, lastLSN)); + } + for (short i = 0; i < tpSize; ++i) { + long transNum = buf.getLong(); + short numPages = buf.getShort(); + List pages = new ArrayList<>(); + for (short j = 0; j < numPages; ++j) { + pages.add(buf.getLong()); + } + touchedPages.put(transNum, pages); + } + return Optional.of(new EndCheckpointLogRecord(dirtyPageTable, transactionTable, touchedPages)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + EndCheckpointLogRecord that = (EndCheckpointLogRecord) o; + return dirtyPageTable.equals(that.dirtyPageTable) && + transactionTable.equals(that.transactionTable) && + touchedPages.equals(that.touchedPages); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), dirtyPageTable, transactionTable, touchedPages); + } + + @Override + public String toString() { + return "EndCheckpointLogRecord{" + + "dirtyPageTable=" + dirtyPageTable + + ", transactionTable=" + transactionTable + + ", touchedPages=" + touchedPages + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/EndTransactionLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/EndTransactionLogRecord.java new file mode 100644 index 0000000..df96f58 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/EndTransactionLogRecord.java @@ -0,0 +1,68 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import java.util.Objects; +import java.util.Optional; + +class EndTransactionLogRecord extends LogRecord { + private long transNum; + private long prevLSN; + + EndTransactionLogRecord(long transNum, long prevLSN) { + super(LogType.END_TRANSACTION); + this.transNum = transNum; + this.prevLSN = prevLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(prevLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long prevLSN = buf.getLong(); + return Optional.of(new EndTransactionLogRecord(transNum, prevLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + EndTransactionLogRecord that = (EndTransactionLogRecord) o; + return transNum == that.transNum && + prevLSN == that.prevLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, prevLSN); + } + + @Override + public String toString() { + return "EndTransactionLogRecord{" + + "transNum=" + transNum + + ", prevLSN=" + prevLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/FreePageLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/FreePageLogRecord.java new file mode 100644 index 0000000..06762a4 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/FreePageLogRecord.java @@ -0,0 +1,113 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; + +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; + +class FreePageLogRecord extends LogRecord { + private long transNum; + private long pageNum; + private long prevLSN; + + FreePageLogRecord(long transNum, long pageNum, long prevLSN) { + super(LogType.FREE_PAGE); + this.transNum = transNum; + this.pageNum = pageNum; + this.prevLSN = prevLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPageNum() { + return Optional.of(pageNum); + } + + @Override + public boolean isUndoable() { + return true; + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public Pair undo(long lastLSN) { + return new Pair<>(new UndoFreePageLogRecord(transNum, pageNum, lastLSN, prevLSN), true); + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + try { + Page p = bm.fetchPage(new DummyLockContext(), pageNum, false); + bm.freePage(p); + p.unpin(); + } catch (NoSuchElementException e) { + /* do nothing - page already freed */ + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(pageNum) + .putLong(prevLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long pageNum = buf.getLong(); + long prevLSN = buf.getLong(); + return Optional.of(new FreePageLogRecord(transNum, pageNum, prevLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + FreePageLogRecord that = (FreePageLogRecord) o; + return transNum == that.transNum && + pageNum == that.pageNum && + prevLSN == that.prevLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, pageNum, prevLSN); + } + + @Override + public String toString() { + return "FreePageLogRecord{" + + "transNum=" + transNum + + ", pageNum=" + pageNum + + ", prevLSN=" + prevLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/FreePartLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/FreePartLogRecord.java new file mode 100644 index 0000000..7f2dad4 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/FreePartLogRecord.java @@ -0,0 +1,109 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; + +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; + +class FreePartLogRecord extends LogRecord { + private long transNum; + private int partNum; + private long prevLSN; + + FreePartLogRecord(long transNum, int partNum, long prevLSN) { + super(LogType.FREE_PART); + this.transNum = transNum; + this.partNum = partNum; + this.prevLSN = prevLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPartNum() { + return Optional.of(partNum); + } + + @Override + public boolean isUndoable() { + return true; + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public Pair undo(long lastLSN) { + return new Pair<>(new UndoFreePartLogRecord(transNum, partNum, lastLSN, prevLSN), true); + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + try { + dsm.freePart(partNum); + } catch (NoSuchElementException e) { + /* do nothing - partition already freed */ + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Integer.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putInt(partNum) + .putLong(prevLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + int partNum = buf.getInt(); + long prevLSN = buf.getLong(); + return Optional.of(new FreePartLogRecord(transNum, partNum, prevLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + FreePartLogRecord that = (FreePartLogRecord) o; + return transNum == that.transNum && + partNum == that.partNum && + prevLSN == that.prevLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, partNum, prevLSN); + } + + @Override + public String toString() { + return "FreePartLogRecord{" + + "transNum=" + transNum + + ", partNum=" + partNum + + ", prevLSN=" + prevLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/LogManager.java b/src/main/java/edu/berkeley/cs186/database/recovery/LogManager.java new file mode 100644 index 0000000..70d48b6 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/LogManager.java @@ -0,0 +1,60 @@ +package edu.berkeley.cs186.database.recovery; + +import java.util.Iterator; + +public interface LogManager extends Iterable, AutoCloseable { + /** + * Writes to the first record in the log. + * @param record log record to replace first record with + */ + void rewriteMasterRecord(MasterLogRecord record); + + /** + * Appends a log record to the log. + * @param record log record to append to the log + * @return LSN of new log record + */ + long appendToLog(LogRecord record); + + /** + * Fetches a specific log record. + * @param LSN LSN of record to fetch + * @return log record with the specified LSN or null if no record found + */ + LogRecord fetchLogRecord(long LSN); + + /** + * Flushes the log to at least the specified record, + * essentially flushing up to and including the page + * that contains the record specified by the LSN. + * @param LSN LSN up to which the log should be flushed + */ + void flushToLSN(long LSN); + + /** + * @return flushedLSN + */ + long getFlushedLSN(); + + /** + * Scan forward in the log from LSN. + * @param LSN LSN to start scanning from + * @return iterator over log entries from LSN + */ + Iterator scanFrom(long LSN); + + /** + * Prints the entire log. For debugging uses only. + */ + void print(); + + /** + * Scan forward in the log from the first record. + * @return iterator over all log entries + */ + @Override + Iterator iterator(); + + @Override + void close(); +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/LogManagerImpl.java b/src/main/java/edu/berkeley/cs186/database/recovery/LogManagerImpl.java new file mode 100644 index 0000000..225b5bc --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/LogManagerImpl.java @@ -0,0 +1,342 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterable; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.common.iterator.ConcatBacktrackingIterator; +import edu.berkeley.cs186.database.common.iterator.IndexBacktrackingIterator; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.PageException; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; + +import java.util.*; + +/** + * The LogManager is responsible for interfacing with the log itself. The log is stored + * on its own partition (partition 0). Since log pages are never deleted, the page number + * is always increasing, so we assign LSNs as follow: + * - page 1: [ LSN 10000, LSN 10040, LSN 10080, ...] + * - page 2: [ LSN 20000, LSN 20030, LSN 20055, ...] + * - page 3: [ LSN 30000, LSN 30047, LSN 30090, ...] + * allowing for up to 10,000 log entries per page. The index (last 4 digits) is the offset + * within the page where the log record starts. Log entries are not fixed width, + * so backwards iteration is not as easy as forward iteration. Page 0 is reserved for the + * master record, which only contains a few log entries: the master record, with LSN 0, followed + * by an empty begin and end checkpoint record. The master record is the only record in the + * entire log that may be rewritten. + * + * The LogManager also is responsible for writing pageLSNs onto pages and flushing the log + * when pages are flushed, and therefore has a few methods that must be called by the buffer + * manager when pages are fetched and evicted (fetchPageHook, fetchNewPageHook, and pageEvictHook). + * These must be called from the buffer manager to ensure that pageLSN is up to date, and + * that flushedLSN >= any pageLSN on disk. + */ +class LogManagerImpl implements LogManager { + private BufferManager bufferManager; + private Deque unflushedLogTail; + private Page logTail; + private Buffer logTailBuffer; + private boolean logTailPinned = false; + private long flushedLSN; + + private static final int LOG_PARTITION = 0; + + LogManagerImpl(BufferManager bufferManager) { + this.bufferManager = bufferManager; + this.unflushedLogTail = new ArrayDeque<>(); + + this.logTail = bufferManager.fetchNewPage(new DummyLockContext(), LOG_PARTITION, true); + this.unflushedLogTail.add(this.logTail); + this.logTailBuffer = this.logTail.getBuffer(); + this.logTail.unpin(); + + this.flushedLSN = maxLSN(this.logTail.getPageNum() - 1L); + } + + /** + * Writes to the first record in the log. + * @param record log record to replace first record with + */ + @Override + public synchronized void rewriteMasterRecord(MasterLogRecord record) { + Page firstPage = bufferManager.fetchPage(new DummyLockContext(), 0L, true); + try { + firstPage.getBuffer().put(record.toBytes()); + firstPage.flush(); + } finally { + firstPage.unpin(); + } + } + + /** + * Appends a log record to the log. + * @param record log record to append to the log + * @return LSN of new log record + */ + @Override + public synchronized long appendToLog(LogRecord record) { + byte[] bytes = record.toBytes(); + // loop in case accessing log tail requires flushing the log in order to evict dirty page to load log tail + do { + if (logTailBuffer == null || bytes.length > DiskSpaceManager.PAGE_SIZE - logTailBuffer.position()) { + logTailPinned = true; + logTail = bufferManager.fetchNewPage(new DummyLockContext(), LOG_PARTITION, true); + unflushedLogTail.add(logTail); + logTailBuffer = logTail.getBuffer(); + } else { + logTailPinned = true; + logTail.pin(); + if (logTailBuffer == null) { + logTail.unpin(); + } + } + } while (logTailBuffer == null); + try { + int pos = logTailBuffer.position(); + logTailBuffer.put(bytes); + long LSN = makeLSN(unflushedLogTail.getLast().getPageNum(), pos); + record.LSN = LSN; + return LSN; + } finally { + logTail.unpin(); + logTailPinned = false; + } + } + + /** + * Fetches a specific log record. + * @param LSN LSN of record to fetch + * @return log record with the specified LSN + */ + @Override + public LogRecord fetchLogRecord(long LSN) { + try { + Page logPage = bufferManager.fetchPage(new DummyLockContext(), getLSNPage(LSN), true); + try { + Buffer buf = logPage.getBuffer(); + buf.position(getLSNIndex(LSN)); + Optional record = LogRecord.fromBytes(buf); + record.ifPresent((LogRecord e) -> e.setLSN(LSN)); + return record.orElse(null); + } finally { + logPage.unpin(); + } + } catch (PageException e) { + return null; + } + } + + /** + * Flushes the log to at least the specified record, + * essentially flushing up to and including the page + * that contains the record specified by the LSN. + * @param LSN LSN up to which the log should be flushed + */ + @Override + public synchronized void flushToLSN(long LSN) { + Iterator iter = unflushedLogTail.iterator(); + long pageNum = getLSNPage(LSN); + while (iter.hasNext()) { + Page page = iter.next(); + if (page.getPageNum() > pageNum) { + break; + } + page.flush(); + iter.remove(); + } + flushedLSN = Math.max(flushedLSN, maxLSN(pageNum)); + if (unflushedLogTail.size() == 0) { + if (!logTailPinned) { + logTail = null; + } + logTailBuffer = null; + } + } + + /** + * @return flushedLSN + */ + @Override + public long getFlushedLSN() { + return flushedLSN; + } + + /** + * Generates LSN from log page number and index + * @param pageNum page number of log page + * @param index index of the log record within the log page + * @return LSN + */ + static long makeLSN(long pageNum, int index) { + return DiskSpaceManager.getPageNum(pageNum) * 10000L + index; + } + + /** + * Generates the max possible LSN on the given page + * @param pageNum page number of log page + * @return max possible LSN on the log page + */ + static long maxLSN(long pageNum) { + return makeLSN(pageNum, 9999); + } + + /** + * Get the page number of the page with the record corresponding to LSN + * @param LSN LSN to get page of + * @return page that LSN resides on + */ + static long getLSNPage(long LSN) { + return LSN / 10000L; + } + + /** + * Get the index within the page of the record corresponding to LSN + * @param LSN LSN to get index of + * @return index in page that LSN resides on + */ + static int getLSNIndex(long LSN) { + return (int) (LSN % 10000L); + } + + /** + * Scan forward in the log from LSN. + * @param LSN LSN to start scanning from + * @return iterator over log entries from LSN + */ + @Override + public Iterator scanFrom(long LSN) { + return new ConcatBacktrackingIterator<>(new LogPagesIterator(LSN)); + } + + @Override + public void print() { + for (LogRecord record : this) { + System.out.println(record); + } + } + + /** + * Scan forward in the log from the first record. + * @return iterator over all log entries + */ + @Override + public Iterator iterator() { + return this.scanFrom(0); + } + + @Override + public synchronized void close() { + if (!this.unflushedLogTail.isEmpty()) { + this.flushToLSN(maxLSN(unflushedLogTail.getLast().getPageNum())); + } + } + + private class LogPageIterator extends IndexBacktrackingIterator { + private Page logPage; + private int startIndex; + + private LogPageIterator(Page logPage, int startIndex) { + super(DiskSpaceManager.PAGE_SIZE); + this.logPage = logPage; + this.startIndex = startIndex; + this.logPage.unpin(); + } + + @Override + protected int getNextNonempty(int currentIndex) { + logPage.pin(); + try { + Buffer buf = logPage.getBuffer(); + if (currentIndex == -1) { + currentIndex = startIndex; + buf.position(currentIndex); + } else { + buf.position(currentIndex); + LogRecord.fromBytes(buf); + currentIndex = buf.position(); + } + + if (LogRecord.fromBytes(buf).isPresent()) { + return currentIndex; + } else { + return DiskSpaceManager.PAGE_SIZE; + } + } finally { + logPage.unpin(); + } + } + + @Override + protected LogRecord getValue(int index) { + logPage.pin(); + try { + Buffer buf = logPage.getBuffer(); + buf.position(index); + LogRecord record = LogRecord.fromBytes(buf).orElseThrow(NoSuchElementException::new); + record.setLSN(makeLSN(logPage.getPageNum(), index)); + return record; + } finally { + logPage.unpin(); + } + } + } + + private class LogPagesIterator implements BacktrackingIterator> { + private BacktrackingIterator nextIter; + private long nextIndex; + + private LogPagesIterator(long startLSN) { + nextIndex = getLSNPage(startLSN); + try { + Page page = bufferManager.fetchPage(new DummyLockContext(), nextIndex, true); + nextIter = new LogPageIterator(page, getLSNIndex(startLSN)); + } catch (PageException e) { + nextIter = null; + } + } + + @Override + public void markPrev() { + throw new UnsupportedOperationException(); + } + + @Override + public void markNext() { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasNext() { + return nextIter != null; + } + + @Override + public BacktrackingIterable next() { + if (hasNext()) { + final BacktrackingIterator iter = nextIter; + BacktrackingIterable iterable = () -> iter; + + nextIter = null; + do { + ++nextIndex; + try { + Page page = bufferManager.fetchPage(new DummyLockContext(), nextIndex, true); + nextIter = new LogPageIterator(page, 0); + } catch (PageException e) { + break; + } + } while (!nextIter.hasNext()); + + return iterable; + } + throw new NoSuchElementException(); + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/LogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/LogRecord.java new file mode 100644 index 0000000..3c79c30 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/LogRecord.java @@ -0,0 +1,252 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.PageException; +import edu.berkeley.cs186.database.memory.BufferManager; + +import java.util.*; +import java.util.function.Consumer; + +/** + * An record of the log. + */ +abstract class LogRecord { + // LSN of this record, or null if not set - this is not actually + // stored on disk, and is only set by the log manager for convenience + Long LSN; + // type of this record + LogType type; + + // method called when redo() is called - only used for testing + private static Consumer onRedo = t -> {}; + + protected LogRecord(LogType type) { + this.type = type; + this.LSN = null; + } + + /** + * @return type of log entry as enumerated in LogType + */ + public final LogType getType() { + return type; + } + + /** + * @return LSN of the log entry + */ + public final long getLSN() { + if (LSN == null) { + throw new IllegalStateException("LSN not set, has this log record been through a log manager call yet?"); + } + return LSN; + } + + /** + * Sets the LSN of a record + * @param LSN LSN intended to assign to a record + */ + final void setLSN(Long LSN) { + this.LSN = LSN; + } + + /** + * Gets the transaction number of a log record, if applicable + * @return optional instance containing transaction number + */ + public Optional getTransNum() { + return Optional.empty(); + } + + /** + * Gets the LSN of the previous record written by the same transaction + * @return optional instance containing prevLSN + */ + public Optional getPrevLSN() { + return Optional.empty(); + } + + /** + * Gets the LSN of record to undo next, if applicable + * @return optional instance containing transaction number + */ + public Optional getUndoNextLSN() { + return Optional.empty(); + } + + /** + * @return optional instance containing page number + * of data that is changed by transaction + */ + public Optional getPageNum() { + return Optional.empty(); + } + + /** + * @return optional instance containing partition number + * of data that is changed by transaction + */ + public Optional getPartNum() { + return Optional.empty(); + } + + public Optional getMaxTransactionNum() { + return Optional.empty(); + } + + /** + * Gets the dirty page table written to the log record, if applicable. + */ + public Map getDirtyPageTable() { + return Collections.emptyMap(); + } + + /** + * Gets the transaction table written to the log record, if applicable. + */ + public Map> getTransactionTable() { + return Collections.emptyMap(); + } + + /** + * Gets the table of transaction numbers mapped to page numbers of + * pages that were touched by the corresponding transaction. + */ + public Map> getTransactionTouchedPages() { + return Collections.emptyMap(); + } + + /** + * @return boolean indicating whether transaction recorded in the + * log record is undoable + */ + public boolean isUndoable() { + return false; + } + + /** + * @return boolean indicating whether transaction recorded in the + * log record is redoable + */ + public boolean isRedoable() { + return false; + } + + /** + * Returns a CLR undoing this log record, but does not execute the undo. + * @param lastLSN lastLSN for the CLR + * @return the CLR corresponding to this log record, and a boolean that is true + * if the log must be flushed up to the CLR after executing the undo, + * and false otherwise. + */ + public Pair undo(long lastLSN) { + throw new UnsupportedOperationException("cannot undo this record: " + this); + } + + /** + * Performs the change described by this log record. + * @param dsm disk space manager + * @param bm buffer manager + */ + public void redo(DiskSpaceManager dsm, BufferManager bm) { + onRedo.accept(this); + if (!isRedoable()) { + throw new UnsupportedOperationException("cannot redo this record: " + this); + } + } + + /** + * Log records are serialized as follows: + * + * - a 1-byte integer indicating the type of log record, followed by + * - a variable number of bytes depending on log record (see specific + * LogRecord implementations for details). + */ + public abstract byte[] toBytes(); + + /** + * Load a log record from a buffer. + * @param buf Buffer containing a serialized log record. + * @return The log record, or Optional.empty() if logType == 0 (marker for no record) + * @throws UnsupportedOperationException if log type is not recognized + */ + public static Optional fromBytes(Buffer buf) { + int type; + try { + type = buf.get(); + } catch (PageException e) { + return Optional.empty(); + } + if (type == 0) { + return Optional.empty(); + } + switch (LogType.fromInt(type)) { + case MASTER: + return MasterLogRecord.fromBytes(buf); + case ALLOC_PAGE: + return AllocPageLogRecord.fromBytes(buf); + case UPDATE_PAGE: + return UpdatePageLogRecord.fromBytes(buf); + case FREE_PAGE: + return FreePageLogRecord.fromBytes(buf); + case ALLOC_PART: + return AllocPartLogRecord.fromBytes(buf); + case FREE_PART: + return FreePartLogRecord.fromBytes(buf); + case COMMIT_TRANSACTION: + return CommitTransactionLogRecord.fromBytes(buf); + case ABORT_TRANSACTION: + return AbortTransactionLogRecord.fromBytes(buf); + case END_TRANSACTION: + return EndTransactionLogRecord.fromBytes(buf); + case BEGIN_CHECKPOINT: + return BeginCheckpointLogRecord.fromBytes(buf); + case END_CHECKPOINT: + return EndCheckpointLogRecord.fromBytes(buf); + case UNDO_ALLOC_PAGE: + return UndoAllocPageLogRecord.fromBytes(buf); + case UNDO_UPDATE_PAGE: + return UndoUpdatePageLogRecord.fromBytes(buf); + case UNDO_FREE_PAGE: + return UndoFreePageLogRecord.fromBytes(buf); + case UNDO_ALLOC_PART: + return UndoAllocPartLogRecord.fromBytes(buf); + case UNDO_FREE_PART: + return UndoFreePartLogRecord.fromBytes(buf); + default: + throw new UnsupportedOperationException("bad log type"); + } + } + + /** + * Set the method called whenever redo() is called on a LogRecord. This + * is only to be used for testing. + * @param handler method to be called whenever redo() is called + */ + static void onRedoHandler(Consumer handler) { + onRedo = handler; + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + LogRecord logRecord = (LogRecord) o; + return type == logRecord.type; + } + + @Override + public int hashCode() { + return Objects.hash(type); + } + + @Override + public String toString() { + return "LogRecord{" + + "type=" + type + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/LogType.java b/src/main/java/edu/berkeley/cs186/database/recovery/LogType.java new file mode 100644 index 0000000..00cb218 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/LogType.java @@ -0,0 +1,51 @@ +package edu.berkeley.cs186.database.recovery; + +enum LogType { + // master log record (stores current checkpoint) + MASTER, + // log record for allocating a new page (via the disk space manager) + ALLOC_PAGE, + // log record for updating part of a page + UPDATE_PAGE, + // log record for freeing a page (via the disk space manager) + FREE_PAGE, + // log record for allocating a new partition (via the disk space manager) + ALLOC_PART, + // log record for freeing a partition (via the disk space manager) + FREE_PART, + // log record for starting a transaction commit + COMMIT_TRANSACTION, + // log record for starting a transaction abort + ABORT_TRANSACTION, + // log record for after a transaction has completely finished + END_TRANSACTION, + // log record for start of a checkpoint + BEGIN_CHECKPOINT, + // log record for finishing a checkpoint; there may be multiple of these + // for a checkpoint + END_CHECKPOINT, + // compensation log record for undoing a page alloc + UNDO_ALLOC_PAGE, + // compensation log record for undoing a page update + UNDO_UPDATE_PAGE, + // compensation log record for undoing a page free + UNDO_FREE_PAGE, + // compensation log record for undoing a partition alloc + UNDO_ALLOC_PART, + // compensation log record for undoing a partition free + UNDO_FREE_PART; + + private static LogType[] values = LogType.values(); + + public int getValue() { + return ordinal() + 1; + } + + public static LogType fromInt(int x) { + if (x < 1 || x > values.length) { + String err = String.format("Unknown TypeId ordinal %d.", x); + throw new IllegalArgumentException(err); + } + return values[x - 1]; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/MasterLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/MasterLogRecord.java new file mode 100644 index 0000000..4278ff3 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/MasterLogRecord.java @@ -0,0 +1,49 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import java.util.Objects; +import java.util.Optional; + +class MasterLogRecord extends LogRecord { + long lastCheckpointLSN; + + MasterLogRecord(long lastCheckpointLSN) { + super(LogType.MASTER); + this.lastCheckpointLSN = lastCheckpointLSN; + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES]; + ByteBuffer.wrap(b).put((byte) getType().getValue()).putLong(lastCheckpointLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + return Optional.of(new MasterLogRecord(buf.getLong())); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + MasterLogRecord that = (MasterLogRecord) o; + return lastCheckpointLSN == that.lastCheckpointLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), lastCheckpointLSN); + } + + @Override + public String toString() { + return "MasterLogRecord{" + + "lastCheckpointLSN=" + lastCheckpointLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/RecoveryManager.java b/src/main/java/edu/berkeley/cs186/database/recovery/RecoveryManager.java new file mode 100644 index 0000000..ae8f338 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/RecoveryManager.java @@ -0,0 +1,173 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; + +/** + * Interface for a recovery manager. + */ +public interface RecoveryManager extends AutoCloseable { + /** + * Initializes the log; only called the first time the database is set up. + */ + void initialize(); + + /** + * Sets the buffer/disk managers. This is not part of the constructor because of the cyclic dependency + * between the buffer manager and recovery manager (the buffer manager must interface with the + * recovery manager to block page evictions until the log has been flushed, but the recovery + * manager needs to interface with the buffer manager to write the log and redo changes). + * @param diskSpaceManager disk space manager + * @param bufferManager buffer manager + */ + void setManagers(DiskSpaceManager diskSpaceManager, BufferManager bufferManager); + + /** + * Called when a new transaction is started. + * @param transaction new transaction + */ + void startTransaction(Transaction transaction); + + /** + * Called when a transaction is about to start committing. + * @param transNum transaction being committed + * @return LSN of the commit record + */ + long commit(long transNum); + + /** + * Called when a transaction is set to be aborted. + * @param transNum transaction being aborted + * @return LSN of the abort record + */ + long abort(long transNum); + + /** + * Called when a transaction is cleaning up; this should roll back + * changes if the transaction is aborting. + * @param transNum transaction to end + * @return LSN of the end record + */ + long end(long transNum); + + /** + * Called before a page is flushed from the buffer cache. This + * method is never called on a log page. + * + * @param pageLSN pageLSN of page about to be flushed + */ + void pageFlushHook(long pageLSN); + + /** + * Called when a page has been updated on disk. + * @param pageNum page number of page updated on disk + */ + void diskIOHook(long pageNum); + + /** + * Called when a write to a page happens. + * + * This method is never called on a log page. Arguments to the before and after params + * must be the same length. + * + * @param transNum transaction performing the write + * @param pageNum page number of page being written + * @param pageOffset offset into page where write begins + * @param before bytes starting at pageOffset before the write + * @param after bytes starting at pageOffset after the write + * @return LSN of last record written to log + */ + long logPageWrite(long transNum, long pageNum, short pageOffset, byte[] before, + byte[] after); + + /** + * Called when a new partition is allocated. A log flush is necessary, + * since changes are visible on disk immediately after this returns. + * + * This method should return -1 if the partition is the log partition. + * + * @param transNum transaction requesting the allocation + * @param partNum partition number of the new partition + * @return LSN of record or -1 if log partition + */ + long logAllocPart(long transNum, int partNum); + + /** + * Called when a partition is freed. A log flush is necessary, + * since changes are visible on disk immediately after this returns. + * + * This method should return -1 if the partition is the log partition. + * + * @param transNum transaction requesting the partition be freed + * @param partNum partition number of the partition being freed + * @return LSN of record or -1 if log partition + */ + long logFreePart(long transNum, int partNum); + + /** + * Called when a new page is allocated. A log flush is necessary, + * since changes are visible on disk immediately after this returns. + * + * This method should return -1 if the page is in the log partition. + * + * @param transNum transaction requesting the allocation + * @param pageNum page number of the new page + * @return LSN of record or -1 if log partition + */ + long logAllocPage(long transNum, long pageNum); + + /** + * Called when a page is freed. A log flush is necessary, + * since changes are visible on disk immediately after this returns. + * + * This method should return -1 if the page is in the log partition. + * + * @param transNum transaction requesting the page be freed + * @param pageNum page number of the page being freed + * @return LSN of record or -1 if log partition + */ + long logFreePage(long transNum, long pageNum); + + /** + * Creates a savepoint for a transaction. Creating a savepoint with + * the same name as an existing savepoint for the transaction should + * delete the old savepoint. + * @param transNum transaction to make savepoint for + * @param name name of savepoint + */ + void savepoint(long transNum, String name); + + /** + * Releases (deletes) a savepoint for a transaction. + * @param transNum transaction to delete savepoint for + * @param name name of savepoint + */ + void releaseSavepoint(long transNum, String name); + + /** + * Rolls back transaction to a savepoint. + * @param transNum transaction to partially rollback + * @param name name of savepoint + */ + void rollbackToSavepoint(long transNum, String name); + + /** + * Creates a checkpoint. + */ + void checkpoint(); + + /** + * Called whenever the database starts up, and performs restart recovery. Recovery is + * complete when the Runnable returned is run to termination. New transactions may be + * started once this method returns. + * @return Runnable to run to finish restart recovery + */ + Runnable restart(); + + /** + * Clean up: log flush, checkpointing, etc. Called when the database is closed. + */ + @Override + void close(); +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/TransactionTableEntry.java b/src/main/java/edu/berkeley/cs186/database/recovery/TransactionTableEntry.java new file mode 100644 index 0000000..defb9d9 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/TransactionTableEntry.java @@ -0,0 +1,65 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.Transaction; + +import java.util.*; + +class TransactionTableEntry { + // Transaction object for the transaction. + Transaction transaction; + // lastLSN of transaction, or 0 if no log entries for the transaction exist. + long lastLSN = 0; + // Set of page numbers of all pages this transaction has modified in some way. + Set touchedPages = new HashSet<>(); + // map of transaction's savepoints + private Map savepoints = new HashMap<>(); + + TransactionTableEntry(Transaction transaction) { + this.transaction = transaction; + } + + void addSavepoint(String name) { + savepoints.put(name, lastLSN); + } + + long getSavepoint(String name) { + if (!savepoints.containsKey(name)) { + throw new NoSuchElementException("transaction " + transaction.getTransNum() + " has no savepoint " + + name); + } + return savepoints.get(name); + } + + void deleteSavepoint(String name) { + if (!savepoints.containsKey(name)) { + throw new NoSuchElementException("transaction " + transaction.getTransNum() + " has no savepoint " + + name); + } + savepoints.remove(name); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + TransactionTableEntry that = (TransactionTableEntry) o; + return lastLSN == that.lastLSN && + Objects.equals(transaction, that.transaction) && + Objects.equals(touchedPages, that.touchedPages) && + Objects.equals(savepoints, that.savepoints); + } + + @Override + public int hashCode() { + return Objects.hash(transaction, lastLSN, touchedPages, savepoints); + } + + @Override + public String toString() { + return "TransactionTableEntry{" + + "transaction=" + transaction + + ", lastLSN=" + lastLSN + + ", touchedPages=" + touchedPages + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/UndoAllocPageLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/UndoAllocPageLogRecord.java new file mode 100644 index 0000000..753b520 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/UndoAllocPageLogRecord.java @@ -0,0 +1,113 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; + +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; + +class UndoAllocPageLogRecord extends LogRecord { + private long transNum; + private long pageNum; + private long prevLSN; + private long undoNextLSN; + + UndoAllocPageLogRecord(long transNum, long pageNum, long prevLSN, long undoNextLSN) { + super(LogType.UNDO_ALLOC_PAGE); + this.transNum = transNum; + this.pageNum = pageNum; + this.prevLSN = prevLSN; + this.undoNextLSN = undoNextLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPageNum() { + return Optional.of(pageNum); + } + + @Override + public Optional getUndoNextLSN() { + return Optional.of(undoNextLSN); + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + try { + Page p = bm.fetchPage(new DummyLockContext(), pageNum, false); + bm.freePage(p); + p.unpin(); + } catch (NoSuchElementException e) { + /* do nothing - page already freed */ + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Long.BYTES + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(pageNum) + .putLong(prevLSN) + .putLong(undoNextLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long pageNum = buf.getLong(); + long prevLSN = buf.getLong(); + long undoNextLSN = buf.getLong(); + return Optional.of(new UndoAllocPageLogRecord(transNum, pageNum, prevLSN, undoNextLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + UndoAllocPageLogRecord that = (UndoAllocPageLogRecord) o; + return transNum == that.transNum && + pageNum == that.pageNum && + prevLSN == that.prevLSN && + undoNextLSN == that.undoNextLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, pageNum, prevLSN, undoNextLSN); + } + + @Override + public String toString() { + return "UndoAllocPageLogRecord{" + + "transNum=" + transNum + + ", pageNum=" + pageNum + + ", prevLSN=" + prevLSN + + ", undoNextLSN=" + undoNextLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/UndoAllocPartLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/UndoAllocPartLogRecord.java new file mode 100644 index 0000000..c307413 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/UndoAllocPartLogRecord.java @@ -0,0 +1,109 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; + +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; + +class UndoAllocPartLogRecord extends LogRecord { + private long transNum; + private int partNum; + private long prevLSN; + private long undoNextLSN; + + UndoAllocPartLogRecord(long transNum, int partNum, long prevLSN, long undoNextLSN) { + super(LogType.UNDO_ALLOC_PART); + this.transNum = transNum; + this.partNum = partNum; + this.prevLSN = prevLSN; + this.undoNextLSN = undoNextLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPartNum() { + return Optional.of(partNum); + } + + @Override + public Optional getUndoNextLSN() { + return Optional.of(undoNextLSN); + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + try { + dsm.freePart(partNum); + } catch (NoSuchElementException e) { + /* do nothing - partition already freed */ + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Integer.BYTES + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putInt(partNum) + .putLong(prevLSN) + .putLong(undoNextLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + int partNum = buf.getInt(); + long prevLSN = buf.getLong(); + long undoNextLSN = buf.getLong(); + return Optional.of(new UndoAllocPartLogRecord(transNum, partNum, prevLSN, undoNextLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + UndoAllocPartLogRecord that = (UndoAllocPartLogRecord) o; + return transNum == that.transNum && + partNum == that.partNum && + prevLSN == that.prevLSN && + undoNextLSN == that.undoNextLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, partNum, prevLSN, undoNextLSN); + } + + @Override + public String toString() { + return "UndoAllocPartLogRecord{" + + "transNum=" + transNum + + ", partNum=" + partNum + + ", prevLSN=" + prevLSN + + ", undoNextLSN=" + undoNextLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/UndoFreePageLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/UndoFreePageLogRecord.java new file mode 100644 index 0000000..0822de9 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/UndoFreePageLogRecord.java @@ -0,0 +1,108 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; + +import java.util.Objects; +import java.util.Optional; + +class UndoFreePageLogRecord extends LogRecord { + private long transNum; + private long pageNum; + private long prevLSN; + private long undoNextLSN; + + UndoFreePageLogRecord(long transNum, long pageNum, long prevLSN, long undoNextLSN) { + super(LogType.UNDO_FREE_PAGE); + this.transNum = transNum; + this.pageNum = pageNum; + this.prevLSN = prevLSN; + this.undoNextLSN = undoNextLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPageNum() { + return Optional.of(pageNum); + } + + @Override + public Optional getUndoNextLSN() { + return Optional.of(undoNextLSN); + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + try { + dsm.allocPage(pageNum); + } catch (IllegalStateException e) { + /* do nothing - page already exists */ + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Long.BYTES + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(pageNum) + .putLong(prevLSN) + .putLong(undoNextLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long pageNum = buf.getLong(); + long prevLSN = buf.getLong(); + long undoNextLSN = buf.getLong(); + return Optional.of(new UndoFreePageLogRecord(transNum, pageNum, prevLSN, undoNextLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + UndoFreePageLogRecord that = (UndoFreePageLogRecord) o; + return transNum == that.transNum && + pageNum == that.pageNum && + prevLSN == that.prevLSN && + undoNextLSN == that.undoNextLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, pageNum, prevLSN, undoNextLSN); + } + + @Override + public String toString() { + return "UndoFreePageLogRecord{" + + "transNum=" + transNum + + ", pageNum=" + pageNum + + ", prevLSN=" + prevLSN + + ", undoNextLSN=" + undoNextLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/UndoFreePartLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/UndoFreePartLogRecord.java new file mode 100644 index 0000000..803c2c3 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/UndoFreePartLogRecord.java @@ -0,0 +1,108 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; + +import java.util.Objects; +import java.util.Optional; + +class UndoFreePartLogRecord extends LogRecord { + private long transNum; + private int partNum; + private long prevLSN; + private long undoNextLSN; + + UndoFreePartLogRecord(long transNum, int partNum, long prevLSN, long undoNextLSN) { + super(LogType.UNDO_FREE_PART); + this.transNum = transNum; + this.partNum = partNum; + this.prevLSN = prevLSN; + this.undoNextLSN = undoNextLSN; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPartNum() { + return Optional.of(partNum); + } + + @Override + public Optional getUndoNextLSN() { + return Optional.of(undoNextLSN); + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + try { + dsm.allocPart(partNum); + } catch (IllegalStateException e) { + /* do nothing - partition already exists */ + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[1 + Long.BYTES + Integer.BYTES + Long.BYTES + Long.BYTES]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putInt(partNum) + .putLong(prevLSN) + .putLong(undoNextLSN); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + int partNum = buf.getInt(); + long prevLSN = buf.getLong(); + long undoNextLSN = buf.getLong(); + return Optional.of(new UndoFreePartLogRecord(transNum, partNum, prevLSN, undoNextLSN)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + UndoFreePartLogRecord that = (UndoFreePartLogRecord) o; + return transNum == that.transNum && + partNum == that.partNum && + prevLSN == that.prevLSN && + undoNextLSN == that.undoNextLSN; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), transNum, partNum, prevLSN, undoNextLSN); + } + + @Override + public String toString() { + return "UndoFreePartLogRecord{" + + "transNum=" + transNum + + ", partNum=" + partNum + + ", prevLSN=" + prevLSN + + ", undoNextLSN=" + undoNextLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/UndoUpdatePageLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/UndoUpdatePageLogRecord.java new file mode 100644 index 0000000..e14ca1e --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/UndoUpdatePageLogRecord.java @@ -0,0 +1,140 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; + +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; + +class UndoUpdatePageLogRecord extends LogRecord { + private long transNum; + private long pageNum; + private long prevLSN; + private long undoNextLSN; + short offset; + byte[] after; + + UndoUpdatePageLogRecord(long transNum, long pageNum, long prevLSN, long undoNextLSN, short offset, + byte[] after) { + super(LogType.UNDO_UPDATE_PAGE); + this.transNum = transNum; + this.pageNum = pageNum; + this.prevLSN = prevLSN; + this.undoNextLSN = undoNextLSN; + this.offset = offset; + this.after = after; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPageNum() { + return Optional.of(pageNum); + } + + @Override + public Optional getUndoNextLSN() { + return Optional.of(undoNextLSN); + } + + @Override + public boolean isRedoable() { + return true; + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + Page page = bm.fetchPage(new DummyLockContext(), pageNum, false); + try { + page.getBuffer().position(offset).put(after); + page.setPageLSN(getLSN()); + } finally { + page.unpin(); + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[(after.length == BufferManager.EFFECTIVE_PAGE_SIZE ? 36 : 37) + after.length]; + Buffer buf = ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(pageNum) + .putLong(prevLSN) + .putLong(undoNextLSN) + .putShort(offset); + // to make sure that the CLR can actually fit on one page... + if (after.length == BufferManager.EFFECTIVE_PAGE_SIZE) { + buf.put((byte) - 1).put(after); + } else { + buf.putShort((short) after.length).put(after); + } + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long pageNum = buf.getLong(); + long prevLSN = buf.getLong(); + long undoNextLSN = buf.getLong(); + short offset = buf.getShort(); + short length = buf.getShort(); + if (length < 0) { + length = BufferManager.EFFECTIVE_PAGE_SIZE; + buf.position(buf.position() - 1); + } + byte[] after = new byte[length]; + buf.get(after); + return Optional.of(new UndoUpdatePageLogRecord(transNum, pageNum, prevLSN, undoNextLSN, offset, + after)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + UndoUpdatePageLogRecord that = (UndoUpdatePageLogRecord) o; + return transNum == that.transNum && + pageNum == that.pageNum && + offset == that.offset && + prevLSN == that.prevLSN && + undoNextLSN == that.undoNextLSN && + Arrays.equals(after, that.after); + } + + @Override + public int hashCode() { + int result = Objects.hash(super.hashCode(), transNum, pageNum, offset, prevLSN, undoNextLSN); + result = 31 * result + Arrays.hashCode(after); + return result; + } + + @Override + public String toString() { + return "UndoUpdatePageLogRecord{" + + "transNum=" + transNum + + ", pageNum=" + pageNum + + ", prevLSN=" + prevLSN + + ", undoNextLSN=" + undoNextLSN + + ", offset=" + offset + + ", after=" + Arrays.toString(after) + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/recovery/UpdatePageLogRecord.java b/src/main/java/edu/berkeley/cs186/database/recovery/UpdatePageLogRecord.java new file mode 100644 index 0000000..ab9fd56 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/recovery/UpdatePageLogRecord.java @@ -0,0 +1,142 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; + +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; + +class UpdatePageLogRecord extends LogRecord { + private long transNum; + private long pageNum; + private long prevLSN; + short offset; + byte[] before; + byte[] after; + + UpdatePageLogRecord(long transNum, long pageNum, long prevLSN, short offset, byte[] before, + byte[] after) { + super(LogType.UPDATE_PAGE); + this.transNum = transNum; + this.pageNum = pageNum; + this.prevLSN = prevLSN; + this.offset = offset; + this.before = before == null ? new byte[0] : before; + this.after = after == null ? new byte[0] : after; + } + + @Override + public Optional getTransNum() { + return Optional.of(transNum); + } + + @Override + public Optional getPrevLSN() { + return Optional.of(prevLSN); + } + + @Override + public Optional getPageNum() { + return Optional.of(pageNum); + } + + @Override + public boolean isUndoable() { + return before.length > 0; + } + + @Override + public boolean isRedoable() { + return after.length > 0; + } + + @Override + public Pair undo(long lastLSN) { + if (!isUndoable()) { + throw new UnsupportedOperationException("cannot undo this record: " + this); + } + return new Pair<>(new UndoUpdatePageLogRecord(transNum, pageNum, lastLSN, prevLSN, offset, before), + false); + } + + @Override + public void redo(DiskSpaceManager dsm, BufferManager bm) { + super.redo(dsm, bm); + + Page page = bm.fetchPage(new DummyLockContext(), pageNum, false); + try { + page.getBuffer().position(offset).put(after); + page.setPageLSN(getLSN()); + } finally { + page.unpin(); + } + } + + @Override + public byte[] toBytes() { + byte[] b = new byte[31 + before.length + after.length]; + ByteBuffer.wrap(b) + .put((byte) getType().getValue()) + .putLong(transNum) + .putLong(pageNum) + .putLong(prevLSN) + .putShort(offset) + .putShort((short) before.length) + .putShort((short) after.length) + .put(before) + .put(after); + return b; + } + + public static Optional fromBytes(Buffer buf) { + long transNum = buf.getLong(); + long pageNum = buf.getLong(); + long prevLSN = buf.getLong(); + short offset = buf.getShort(); + byte[] before = new byte[buf.getShort()]; + byte[] after = new byte[buf.getShort()]; + buf.get(before).get(after); + return Optional.of(new UpdatePageLogRecord(transNum, pageNum, prevLSN, offset, before, after)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { return false; } + UpdatePageLogRecord that = (UpdatePageLogRecord) o; + return transNum == that.transNum && + pageNum == that.pageNum && + offset == that.offset && + prevLSN == that.prevLSN && + Arrays.equals(before, that.before) && + Arrays.equals(after, that.after); + } + + @Override + public int hashCode() { + int result = Objects.hash(super.hashCode(), transNum, pageNum, offset, prevLSN); + result = 31 * result + Arrays.hashCode(before); + result = 31 * result + Arrays.hashCode(after); + return result; + } + + @Override + public String toString() { + return "UpdatePageLogRecord{" + + "transNum=" + transNum + + ", pageNum=" + pageNum + + ", offset=" + offset + + ", before=" + Arrays.toString(before) + + ", after=" + Arrays.toString(after) + + ", prevLSN=" + prevLSN + + ", LSN=" + LSN + + '}'; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/HeapFile.java b/src/main/java/edu/berkeley/cs186/database/table/HeapFile.java new file mode 100644 index 0000000..20fbb93 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/HeapFile.java @@ -0,0 +1,65 @@ +package edu.berkeley.cs186.database.table; + +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterable; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.memory.Page; + +/** + * Interface for a heap file, which receives requests for pages with + * a certain amount of space, and returns a page with enough space. + * Assumes a packed page layout. + */ +public interface HeapFile extends BacktrackingIterable { + /** + * @return effective page size (including metadata). + */ + short getEffectivePageSize(); + + /** + * Sets the size of metadata on an empty page. + * @param emptyPageMetadataSize amount of metadata on empty page + */ + void setEmptyPageMetadataSize(short emptyPageMetadataSize); + + /** + * Fetches a specific pinned page. + * @param pageNum page number + * @return the pinned page + */ + Page getPage(long pageNum); + + /** + * Fetches a data page with a certain amount of unused space. New data and + * header pages may be allocated as necessary. + * @param requiredSpace amount of space needed; + * cannot be larger than effectivePageSize - emptyPageMetadataSize + * @return pinned page with the requested area of contiguous space + */ + Page getPageWithSpace(short requiredSpace); + + /** + * Updates the amount of free space on a page. Updating to effectivePageSize + * frees the page, and it may no longer be used. + * @param page the data page returned by getPageWithSpace + * @param newFreeSpace the size (in bytes) of the free space on the page + */ + void updateFreeSpace(Page page, short newFreeSpace); + + /** + * @return iterator of all allocated data pages + */ + @Override + BacktrackingIterator iterator(); + + /** + * Returns estimate of number of data pages. + * @return estimate of number of data pages + */ + int getNumDataPages(); + + /** + * Gets partition number of partition the heap file lies on. + * @return partition number + */ + int getPartNum(); +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/MarkerRecord.java b/src/main/java/edu/berkeley/cs186/database/table/MarkerRecord.java new file mode 100644 index 0000000..63d328c --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/MarkerRecord.java @@ -0,0 +1,19 @@ +package edu.berkeley.cs186.database.table; + +import java.util.ArrayList; + +/** + * An empty record used to delineate groups in the GroupByOperator (see + * comments in GroupByOperator). + */ +public class MarkerRecord extends Record { + private static final MarkerRecord record = new MarkerRecord(); + + private MarkerRecord() { + super(new ArrayList<>()); + } + + public static MarkerRecord getMarker() { + return MarkerRecord.record; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/PageDirectory.java b/src/main/java/edu/berkeley/cs186/database/table/PageDirectory.java new file mode 100644 index 0000000..7743701 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/PageDirectory.java @@ -0,0 +1,494 @@ +package edu.berkeley.cs186.database.table; + +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterable; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.iterator.ConcatBacktrackingIterator; +import edu.berkeley.cs186.database.common.iterator.IndexBacktrackingIterator; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.concurrency.LockType; +import edu.berkeley.cs186.database.concurrency.LockUtil; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.PageException; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.Page; + +import java.util.NoSuchElementException; +import java.util.Random; + +/** + * An implementation of a heap file, using a page directory. Assumes data pages are packed (but record + * lengths do not need to be fixed-length). + * + * Header pages are layed out as follows: + * - first byte: 0x1 to indicate valid allocated page + * - next 4 bytes: page directory id + * - next 8 bytes: page number of next header page, or -1 (0xFFFFFFFFFFFFFFFF) if no next header page. + * - next 10 bytes: page number of data page (or -1), followed by 2 bytes of amount of free space + * - repeat 10 byte entries + * + * Data pages contain a small header containing: + * - 4-byte page directory id + * - 4-byte index of which header page manages it + * - 2-byte offset indicating which slot in the header page its data page entry resides + * + * This header is used to quickly locate and update the header page when the amount of free space on the data page + * changes, as well as ensure that we do not modify pages in other page directories by accident. + * + * The page directory id is a randomly generated 32-bit integer used to help detect bugs (where we attempt + * to write to a page that is not managed by the page directory). + */ +public class PageDirectory implements HeapFile { + // size of the header in header pages + private static final short HEADER_HEADER_SIZE = 13; + + // number of data page entries in a header page + private static final short HEADER_ENTRY_COUNT = (BufferManager.EFFECTIVE_PAGE_SIZE - + HEADER_HEADER_SIZE) / DataPageEntry.SIZE; + + // size of the header in data pages + private static final short DATA_HEADER_SIZE = 10; + + // effective page size + private static final short EFFECTIVE_PAGE_SIZE = BufferManager.EFFECTIVE_PAGE_SIZE - + DATA_HEADER_SIZE; + + // the buffer manager + private BufferManager bufferManager; + + // partition to allocate new header pages in - may be different from partition + // for data pages + private int partNum; + + // First header page + private HeaderPage firstHeader; + + // Size of metadata of an empty data page. + private short emptyPageMetadataSize; + + // lock context of heap file/table + private LockContext lockContext; + + // page directory id + private int pageDirectoryId; + + /** + * Creates a new heap file, or loads existing file if one already + * exists at partNum. + * @param bufferManager buffer manager + * @param partNum partition to allocate new header pages in (can be different partition + * from data pages) + * @param pageNum first header page of heap file + * @param emptyPageMetadataSize size of metadata on an empty page + * @param lockContext lock context of this heap file + */ + public PageDirectory(BufferManager bufferManager, int partNum, long pageNum, + short emptyPageMetadataSize, LockContext lockContext) { + // TODO(hw4_part2): update table capacity + this.bufferManager = bufferManager; + this.partNum = partNum; + this.emptyPageMetadataSize = emptyPageMetadataSize; + this.lockContext = lockContext; + this.firstHeader = new HeaderPage(pageNum, 0, true); + } + + @Override + public short getEffectivePageSize() { + return EFFECTIVE_PAGE_SIZE; + } + + @Override + public void setEmptyPageMetadataSize(short emptyPageMetadataSize) { + this.emptyPageMetadataSize = emptyPageMetadataSize; + } + + @Override + public Page getPage(long pageNum) { + return new DataPage(pageDirectoryId, this.bufferManager.fetchPage(lockContext, pageNum, false)); + } + + @Override + public Page getPageWithSpace(short requiredSpace) { + // TODO(hw4_part2): modify for smarter locking + + if (requiredSpace <= 0) { + throw new IllegalArgumentException("cannot request nonpositive amount of space"); + } + if (requiredSpace > EFFECTIVE_PAGE_SIZE - emptyPageMetadataSize) { + throw new IllegalArgumentException("requesting page with more space than the size of the page"); + } + + Page page = this.firstHeader.loadPageWithSpace(requiredSpace); + + return new DataPage(pageDirectoryId, page); + } + + @Override + public void updateFreeSpace(Page page, short newFreeSpace) { + if (newFreeSpace <= 0 || newFreeSpace > EFFECTIVE_PAGE_SIZE - emptyPageMetadataSize) { + throw new IllegalArgumentException("bad size for data page free space"); + } + + int headerIndex; + short offset; + page.pin(); + try { + Buffer b = ((DataPage) page).getFullBuffer(); + b.position(4); // skip page directory id + headerIndex = b.getInt(); + offset = b.getShort(); + } finally { + page.unpin(); + } + + HeaderPage headerPage = firstHeader; + for (int i = 0; i < headerIndex; ++i) { + headerPage = headerPage.nextPage; + } + headerPage.updateSpace(page, offset, newFreeSpace); + } + + @Override + public BacktrackingIterator iterator() { + return new ConcatBacktrackingIterator<>(new HeaderPageIterator()); + } + + @Override + public int getNumDataPages() { + int numDataPages = 0; + HeaderPage headerPage = firstHeader; + while (headerPage != null) { + numDataPages += headerPage.numDataPages; + headerPage = headerPage.nextPage; + } + return numDataPages; + } + + @Override + public int getPartNum() { + return partNum; + } + + /** + * Wrapper around page object to skip the header and verify that it belongs to this + * page directory. + */ + private static class DataPage extends Page { + private DataPage(int pageDirectoryId, Page page) { + super(page); + + Buffer buffer = super.getBuffer(); + if (buffer.getInt() != pageDirectoryId) { + page.unpin(); + throw new PageException("data page directory id does not match"); + } + } + + @Override + public Buffer getBuffer() { + return super.getBuffer().position(DATA_HEADER_SIZE).slice(); + } + + // get the full buffer (without skipping header) for internal use + private Buffer getFullBuffer() { + return super.getBuffer(); + } + } + + /** + * Entry for a data page inside a header page. + */ + private static class DataPageEntry { + // size in bytes of entry + private static final int SIZE = 10; + + // page number of data page + private long pageNum; + + // size in bytes of free space in data page + private short freeSpace; + + // creates an invalid data page entry (one where no data page has been allocated yet). + private DataPageEntry() { + this(DiskSpaceManager.INVALID_PAGE_NUM, (short) - 1); + } + + private DataPageEntry(long pageNum, short freeSpace) { + this.pageNum = pageNum; + this.freeSpace = freeSpace; + } + + // returns if data page entry refers to a valid data page + private boolean isValid() { + return this.pageNum != DiskSpaceManager.INVALID_PAGE_NUM; + } + + private void toBytes(Buffer b) { + b.putLong(pageNum).putShort(freeSpace); + } + + private static DataPageEntry fromBytes(Buffer b) { + return new DataPageEntry(b.getLong(), b.getShort()); + } + + @Override + public String toString() { + return "[Page " + pageNum + ", " + freeSpace + " free]"; + } + } + + /** + * Represents a single header page. + */ + private class HeaderPage implements BacktrackingIterable { + private HeaderPage nextPage; + private Page page; + private short numDataPages; + private int headerOffset; + + private HeaderPage(long pageNum, int headerOffset, boolean firstHeader) { + this.page = bufferManager.fetchPage(lockContext, pageNum, false); + // We do not lock header pages for the entirety of the transaction. Instead, we simply + // use the buffer frame lock (from pinning) to ensure that one transaction writes at a time. + // This does mean that we do not have complete isolation in the header pages, but this does not + // really matter, as the only observable effect is that a transaction may be told to use a different + // data page, which is perfectly fine. + this.page.disableLocking(); + this.numDataPages = 0; + long nextPageNum; + try { + Buffer pageBuffer = this.page.getBuffer(); + if (pageBuffer.get() != (byte) 1) { + byte[] buf = new byte[BufferManager.EFFECTIVE_PAGE_SIZE]; + Buffer b = ByteBuffer.wrap(buf); + // invalid page, initialize empty header page + if (firstHeader) { + pageDirectoryId = new Random().nextInt(); + } + b.position(0).put((byte) 1).putInt(pageDirectoryId).putLong(DiskSpaceManager.INVALID_PAGE_NUM); + DataPageEntry invalidPageEntry = new DataPageEntry(); + for (int i = 0; i < HEADER_ENTRY_COUNT; ++i) { + invalidPageEntry.toBytes(b); + } + nextPageNum = -1L; + + pageBuffer.put(buf, 0, buf.length); + } else { + // load header page + if (firstHeader) { + pageDirectoryId = pageBuffer.getInt(); + } else if (pageDirectoryId != pageBuffer.getInt()) { + throw new PageException("header page page directory id does not match"); + } + nextPageNum = pageBuffer.getLong(); + for (int i = 0; i < HEADER_ENTRY_COUNT; ++i) { + DataPageEntry dpe = DataPageEntry.fromBytes(pageBuffer); + if (dpe.isValid()) { + ++this.numDataPages; + } + } + } + } finally { + this.page.unpin(); + } + this.headerOffset = headerOffset; + if (nextPageNum == DiskSpaceManager.INVALID_PAGE_NUM) { + this.nextPage = null; + } else { + this.nextPage = new HeaderPage(nextPageNum, headerOffset + 1, false); + } + } + + // add a new header page + private void addNewHeaderPage() { + if (this.nextPage != null) { + this.nextPage.addNewHeaderPage(); + return; + } + Page page = bufferManager.fetchNewPage(lockContext, partNum, false); + this.page.pin(); + try { + this.nextPage = new HeaderPage(page.getPageNum(), headerOffset + 1, false); + this.page.getBuffer().position(1).putLong(page.getPageNum()); + } finally { + this.page.unpin(); + page.unpin(); + } + } + + // gets and loads a page with the required free space + private Page loadPageWithSpace(short requiredSpace) { + // TODO(hw4_part2): update table capacity + + this.page.pin(); + try { + Buffer b = this.page.getBuffer(); + b.position(HEADER_HEADER_SIZE); + + // if we have any data page managed by this header page with enough space, return it + short unusedSlot = -1; + for (short i = 0; i < HEADER_ENTRY_COUNT; ++i) { + DataPageEntry dpe = DataPageEntry.fromBytes(b); + if (!dpe.isValid()) { + if (unusedSlot == -1) { + unusedSlot = i; + } + continue; + } + if (dpe.freeSpace >= requiredSpace) { + dpe.freeSpace -= requiredSpace; + b.position(b.position() - DataPageEntry.SIZE); + dpe.toBytes(b); + + return bufferManager.fetchPage(lockContext, dpe.pageNum, false); + } + } + + // if we have any unused slot in this header page, allocate a new data page + if (unusedSlot != -1) { + Page page = bufferManager.fetchNewPage(lockContext, partNum, false); + DataPageEntry dpe = new DataPageEntry(page.getPageNum(), + (short) (EFFECTIVE_PAGE_SIZE - emptyPageMetadataSize - requiredSpace)); + + b.position(HEADER_HEADER_SIZE + DataPageEntry.SIZE * unusedSlot); + dpe.toBytes(b); + + page.getBuffer().putInt(pageDirectoryId).putInt(headerOffset).putShort(unusedSlot); + + ++this.numDataPages; + + return page; + } + + // if we have no next header page, make one + if (this.nextPage == null) { + this.addNewHeaderPage(); + } + + // no space on this header page, try next one + return this.nextPage.loadPageWithSpace(requiredSpace); + } finally { + this.page.unpin(); + } + } + + // updates free space + private void updateSpace(Page dataPage, short index, short newFreeSpace) { + this.page.pin(); + try { + if (newFreeSpace < EFFECTIVE_PAGE_SIZE - emptyPageMetadataSize) { + // write new free space to disk + Buffer b = this.page.getBuffer(); + b.position(HEADER_HEADER_SIZE + DataPageEntry.SIZE * index); + DataPageEntry dpe = DataPageEntry.fromBytes(b); + dpe.freeSpace = newFreeSpace; + b.position(HEADER_HEADER_SIZE + DataPageEntry.SIZE * index); + dpe.toBytes(b); + } else { + // the entire page is free; free it + Buffer b = this.page.getBuffer(); + b.position(HEADER_HEADER_SIZE + DataPageEntry.SIZE * index); + (new DataPageEntry()).toBytes(b); + bufferManager.freePage(dataPage); + } + } finally { + this.page.unpin(); + } + } + + @Override + public BacktrackingIterator iterator() { + return new HeaderPageIterator(); + } + + // iterator over the data pages managed by this header page + private class HeaderPageIterator extends IndexBacktrackingIterator { + private HeaderPageIterator() { + super(HEADER_ENTRY_COUNT); + } + + @Override + protected int getNextNonempty(int currentIndex) { + HeaderPage.this.page.pin(); + try { + Buffer b = HeaderPage.this.page.getBuffer(); + b.position(HEADER_HEADER_SIZE + DataPageEntry.SIZE * ++currentIndex); + for (int i = currentIndex; i < HEADER_ENTRY_COUNT; ++i) { + DataPageEntry dpe = DataPageEntry.fromBytes(b); + if (dpe.isValid()) { + return i; + } + } + return HEADER_ENTRY_COUNT; + } finally { + HeaderPage.this.page.unpin(); + } + } + + @Override + protected Page getValue(int index) { + HeaderPage.this.page.pin(); + try { + Buffer b = HeaderPage.this.page.getBuffer(); + b.position(HEADER_HEADER_SIZE + DataPageEntry.SIZE * index); + DataPageEntry dpe = DataPageEntry.fromBytes(b); + return new DataPage(pageDirectoryId, bufferManager.fetchPage(lockContext, dpe.pageNum, false)); + } finally { + HeaderPage.this.page.unpin(); + } + } + } + } + + /** + * Iterator over header pages. + */ + private class HeaderPageIterator implements BacktrackingIterator> { + private HeaderPage nextPage; + private HeaderPage prevPage; + private HeaderPage markedPage; + + private HeaderPageIterator() { + this.nextPage = firstHeader; + this.prevPage = null; + this.markedPage = null; + } + + @Override + public boolean hasNext() { + return this.nextPage != null; + } + + @Override + public HeaderPage next() { + if (!this.hasNext()) { + throw new NoSuchElementException(); + } + HeaderPage next = this.nextPage; + this.prevPage = next; + this.nextPage = next.nextPage; + return next; + } + + @Override + public void markPrev() { + if (this.prevPage != null) { + this.markedPage = this.prevPage; + } + } + + @Override + public void markNext() { + this.markedPage = this.nextPage; + } + + @Override + public void reset() { + if (this.markedPage != null) { + this.prevPage = null; + this.nextPage = this.markedPage; + } + } + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/Record.java b/src/main/java/edu/berkeley/cs186/database/table/Record.java new file mode 100644 index 0000000..2eac36e --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/Record.java @@ -0,0 +1,68 @@ +package edu.berkeley.cs186.database.table; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.Type; + +/** A Record is just list of DataBoxes. */ +public class Record { + private List values; + + public Record(List values) { + this.values = values; + } + + public List getValues() { + return this.values; + } + + public byte[] toBytes(Schema schema) { + ByteBuffer byteBuffer = ByteBuffer.allocate(schema.getSizeInBytes()); + for (DataBox value : values) { + byteBuffer.put(value.toBytes()); + } + return byteBuffer.array(); + } + + /** + * Takes a byte[] and decodes it into a Record. This method assumes that the + * input byte[] represents a record that corresponds to this schema. + * + * @param buf the byte array to decode + * @param schema the schema used for this record + * @return the decoded Record + */ + public static Record fromBytes(Buffer buf, Schema schema) { + List values = new ArrayList<>(); + for (Type t : schema.getFieldTypes()) { + values.add(DataBox.fromBytes(buf, t)); + } + return new Record(values); + } + + @Override + public String toString() { + return values.toString(); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Record)) { + return false; + } + Record r = (Record) o; + return values.equals(r.values); + } + + @Override + public int hashCode() { + return values.hashCode(); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/RecordId.java b/src/main/java/edu/berkeley/cs186/database/table/RecordId.java new file mode 100644 index 0000000..29345bb --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/RecordId.java @@ -0,0 +1,80 @@ +package edu.berkeley.cs186.database.table; + +import edu.berkeley.cs186.database.common.Buffer; + +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * A Record in a particular table is uniquely identified by its page number + * (the number of the page on which it resides) and its entry number (the + * record's index in the page). A RecordId is a pair of the page number and + * entry number. + */ +public class RecordId implements Comparable { + private long pageNum; + private short entryNum; + + public RecordId(long pageNum, short entryNum) { + this.pageNum = pageNum; + this.entryNum = entryNum; + } + + public long getPageNum() { + return this.pageNum; + } + + public short getEntryNum() { + return this.entryNum; + } + + public static int getSizeInBytes() { + // See toBytes. + return Long.BYTES + Short.BYTES; + } + + public byte[] toBytes() { + // A RecordId is serialized as its 4-byte page number followed by its + // 2-byte short. + return ByteBuffer.allocate(getSizeInBytes()) + .putLong(pageNum) + .putShort(entryNum) + .array(); + } + + public static RecordId fromBytes(Buffer buf) { + return new RecordId(buf.getLong(), buf.getShort()); + } + + @Override + public String toString() { + return String.format("RecordId(%d, %d)", pageNum, entryNum); + } + + public String toSexp() { + return String.format("(%d %d)", pageNum, entryNum); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof RecordId)) { + return false; + } + RecordId r = (RecordId) o; + return pageNum == r.pageNum && entryNum == r.entryNum; + } + + @Override + public int hashCode() { + return Objects.hash(pageNum, entryNum); + } + + @Override + public int compareTo(RecordId r) { + int x = Long.compare(pageNum, r.pageNum); + return x == 0 ? Integer.compare(entryNum, r.entryNum) : x; + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/RecordIterator.java b/src/main/java/edu/berkeley/cs186/database/table/RecordIterator.java new file mode 100644 index 0000000..65555bd --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/RecordIterator.java @@ -0,0 +1,68 @@ +package edu.berkeley.cs186.database.table; + +import java.util.Iterator; + +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.DatabaseException; + +/** + * A RecordIterator wraps an Iterator to form an BacktrackingIterator. + * For example, + * + * Iterator ridIterator = t.ridIterator(); + * RecordIterator recordIterator = new RecordIterator(t, ridIterator); + * recordIterator.next(); // equivalent to t.getRecord(ridIterator.next()) + * recordIterator.next(); // equivalent to t.getRecord(ridIterator.next()) + * recordIterator.next(); // equivalent to t.getRecord(ridIterator.next()) + */ +public class RecordIterator implements BacktrackingIterator { + private Iterator ridIter; + private Table table; + + public RecordIterator(Table table, Iterator ridIter) { + this.ridIter = ridIter; + this.table = table; + } + + @Override + public boolean hasNext() { + return ridIter.hasNext(); + } + + @Override + public Record next() { + try { + return table.getRecord(ridIter.next()); + } catch (DatabaseException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void markPrev() { + if (ridIter instanceof BacktrackingIterator) { + ((BacktrackingIterator) ridIter).markPrev(); + } else { + throw new UnsupportedOperationException("Cannot markPrev using underlying iterator"); + } + } + + @Override + public void markNext() { + if (ridIter instanceof BacktrackingIterator) { + ((BacktrackingIterator) ridIter).markNext(); + } else { + throw new UnsupportedOperationException("Cannot markNext using underlying iterator"); + } + } + + @Override + public void reset() { + if (ridIter instanceof BacktrackingIterator) { + ((BacktrackingIterator) ridIter).reset(); + } else { + throw new UnsupportedOperationException("Cannot reset using underlying iterator"); + } + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/table/Schema.java b/src/main/java/edu/berkeley/cs186/database/table/Schema.java new file mode 100644 index 0000000..af69061 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/Schema.java @@ -0,0 +1,143 @@ +package edu.berkeley.cs186.database.table; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.Type; + +/** + * The schema of a table includes the name and type of every one of its + * fields. For example, the following schema: + * + * List fieldNames = Arrays.asList("x", "y"); + * List fieldTypes = Arrays.asList(Type.intType(), Type.floatType()); + * Schema s = new Schema(fieldNames, fieldSize); + * + * represents a table with an int field named "x" and a float field named "y". + */ +public class Schema { + private List fieldNames; + private List fieldTypes; + private short sizeInBytes; + + public Schema(List fieldNames, List fieldTypes) { + assert(fieldNames.size() == fieldTypes.size()); + this.fieldNames = fieldNames; + this.fieldTypes = fieldTypes; + + sizeInBytes = 0; + for (Type t : fieldTypes) { + sizeInBytes += t.getSizeInBytes(); + } + } + + public List getFieldNames() { + return fieldNames; + } + + public List getFieldTypes() { + return fieldTypes; + } + + public short getSizeInBytes() { + return sizeInBytes; + } + + Record verify(List values) { + if (values.size() != fieldNames.size()) { + String err = String.format("Expected %d values, but got %d.", + fieldNames.size(), values.size()); + throw new DatabaseException(err); + } + + for (int i = 0; i < values.size(); ++i) { + Type actual = values.get(i).type(); + Type expected = fieldTypes.get(i); + if (!actual.equals(expected)) { + String err = String.format( + "Expected field %d to be of type %s, but got value of type %s.", + i, expected, actual); + throw new DatabaseException(err); + } + } + + return new Record(values); + } + + public byte[] toBytes() { + // A schema is serialized as follows. We first write the number of fields + // (4 bytes). Then, for each field, we write + // + // 1. the length of the field name (4 bytes), + // 2. the field's name, + // 3. and the field's type. + + // First, we compute the number of bytes we need to serialize the schema. + int size = Integer.BYTES; // The length of the schema. + for (int i = 0; i < fieldNames.size(); ++i) { + size += Integer.BYTES; // The length of the field name. + size += fieldNames.get(i).length(); // The field name. + size += fieldTypes.get(i).toBytes().length; // The type. + } + + // Then we serialize it. + ByteBuffer buf = ByteBuffer.allocate(size); + buf.putInt(fieldNames.size()); + for (int i = 0; i < fieldNames.size(); ++i) { + buf.putInt(fieldNames.get(i).length()); + buf.put(fieldNames.get(i).getBytes(Charset.forName("UTF-8"))); + buf.put(fieldTypes.get(i).toBytes()); + } + return buf.array(); + } + + public static Schema fromBytes(Buffer buf) { + int size = buf.getInt(); + List fieldNames = new ArrayList<>(); + List fieldTypes = new ArrayList<>(); + for (int i = 0; i < size; ++i) { + int fieldSize = buf.getInt(); + byte[] bytes = new byte[fieldSize]; + buf.get(bytes); + fieldNames.add(new String(bytes, Charset.forName("UTF-8"))); + fieldTypes.add(Type.fromBytes(buf)); + } + return new Schema(fieldNames, fieldTypes); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("("); + for (int i = 0; i < fieldNames.size(); ++i) { + sb.append(String.format("%s: %s", fieldNames.get(i), fieldTypes.get(i))); + if (i != fieldNames.size()) { + sb.append(", "); + } + } + sb.append(")"); + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Schema)) { + return false; + } + Schema s = (Schema) o; + return fieldNames.equals(s.fieldNames) && fieldTypes.equals(s.fieldTypes); + } + + @Override + public int hashCode() { + return Objects.hash(fieldNames, fieldTypes); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/Table.java b/src/main/java/edu/berkeley/cs186/database/table/Table.java new file mode 100644 index 0000000..280359f --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/Table.java @@ -0,0 +1,575 @@ +package edu.berkeley.cs186.database.table; + +import java.util.*; + +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.common.iterator.*; +import edu.berkeley.cs186.database.common.Bits; +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.concurrency.LockType; +import edu.berkeley.cs186.database.concurrency.LockUtil; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.io.PageException; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.table.stats.TableStats; + +/** + * # Overview + * A Table represents a database table with which users can insert, get, + * update, and delete records: + * + * // Create a brand new table t(x: int, y: int) which is persisted in the + * // the heap file heapFile. + * List fieldNames = Arrays.asList("x", "y"); + * List fieldTypes = Arrays.asList(Type.intType(), Type.intType()); + * Schema schema = new Schema(fieldNames, fieldTypes); + * Table t = new Table("t", schema, heapFile, new DummyLockContext()); + * + * // Insert, get, update, and delete records. + * List a = Arrays.asList(new IntDataBox(1), new IntDataBox(2)); + * List b = Arrays.asList(new IntDataBox(3), new IntDataBox(4)); + * RecordId rid = t.addRecord(a); + * Record ra = t.getRecord(rid); + * t.updateRecord(b, rid); + * Record rb = t.getRecord(rid); + * t.deleteRecord(rid); + * + * # Persistence + * Every table is persisted in its own HeapFile object (passed into the constructor), + * which interfaces with the BufferManager and DiskSpaceManager to save it to disk. + * + * A table can be loaded again by simply constructing it with the same parameters. + * + * # Storage Format + * Now, we discuss how tables serialize their data. + * + * All pages are data pages - there are no header pages, because all metadata is + * stored elsewhere (as rows in the information_schema.tables table). Every data + * page begins with a n-byte bitmap followed by m records. The bitmap indicates + * which records in the page are valid. The values of n and m are set to maximize the + * number of records per page (see computeDataPageNumbers for details). + * + * For example, here is a cartoon of what a table's file would look like if we + * had 5-byte pages and 1-byte records: + * + * +----------+----------+----------+----------+----------+ \ + * Page 0 | 1001xxxx | 01111010 | xxxxxxxx | xxxxxxxx | 01100001 | | + * +----------+----------+----------+----------+----------+ | + * Page 1 | 1101xxxx | 01110010 | 01100100 | xxxxxxxx | 01101111 | |- data + * +----------+----------+----------+----------+----------+ | + * Page 2 | 0011xxxx | xxxxxxxx | xxxxxxxx | 01111010 | 00100001 | | + * +----------+----------+----------+----------+----------+ / + * \________/ \________/ \________/ \________/ \________/ + * bitmap record 0 record 1 record 2 record 3 + * + * - The first page (Page 0) is a data page. The first byte of this data page + * is a bitmap, and the next four bytes are each records. The first and + * fourth bit are set indicating that record 0 and record 3 are valid. + * Record 1 and record 2 are invalid, so we ignore their contents. + * Similarly, the last four bits of the bitmap are unused, so we ignore + * their contents. + * - The second and third page (Page 1 and 2) are also data pages and are + * formatted similar to Page 0. + * + * When we add a record to a table, we add it to the very first free slot in + * the table. See addRecord for more information. + * + * Some tables have large records. In order to efficiently handle tables with + * large records (that still fit on a page), we format these tables a bit differently, + * by giving each record a full page. Tables with full page records do not have a bitmap. + * Instead, each allocated page is a single record, and we indicate that a page does + * not contain a record by simply freeing the page. + * + * In some cases, this behavior may be desirable even for small records (our database + * only supports locking at the page level, so in cases where tuple-level locks are + * necessary even at the cost of an I/O per tuple, a full page record may be desirable), + * and may be explicitly toggled on with the setFullPageRecords method. + */ +public class Table implements BacktrackingIterable { + // The name of the table. + private String name; + + // The schema of the table. + private Schema schema; + + // The page directory persisting the table. + private HeapFile heapFile; + + // The size (in bytes) of the bitmap found at the beginning of each data page. + private int bitmapSizeInBytes; + + // The number of records on each data page. + private int numRecordsPerPage; + + // Statistics about the contents of the database. + private TableStats stats; + + // The number of records in the table. + private long numRecords; + + // The lock context of the table. + private LockContext lockContext; + + // Constructors ////////////////////////////////////////////////////////////// + /** + * Load a table named `name` with schema `schema` from `heapFile`. `lockContext` + * is the lock context of the table (use a DummyLockContext() to disable locking). A + * new table will be created if none exists on the heapfile. + */ + public Table(String name, Schema schema, HeapFile heapFile, LockContext lockContext) { + // TODO(hw4_part2): table locking code + + this.name = name; + this.heapFile = heapFile; + this.schema = schema; + this.bitmapSizeInBytes = computeBitmapSizeInBytes(heapFile.getEffectivePageSize(), schema); + this.numRecordsPerPage = computeNumRecordsPerPage(heapFile.getEffectivePageSize(), schema); + // mark everything that is not used for records as metadata + this.heapFile.setEmptyPageMetadataSize((short) (heapFile.getEffectivePageSize() - numRecordsPerPage + * schema.getSizeInBytes())); + + this.stats = new TableStats(this.schema, this.numRecordsPerPage); + this.numRecords = 0; + + Iterator iter = this.heapFile.iterator(); + while(iter.hasNext()) { + Page page = iter.next(); + byte[] bitmap = getBitMap(page); + + for (short i = 0; i < numRecordsPerPage; ++i) { + if (Bits.getBit(bitmap, i) == Bits.Bit.ONE) { + Record r = getRecord(new RecordId(page.getPageNum(), i)); + stats.addRecord(r); + numRecords++; + } + } + page.unpin(); + } + + this.lockContext = lockContext; + } + + // Accessors ///////////////////////////////////////////////////////////////// + public String getName() { + return name; + } + + public Schema getSchema() { + return schema; + } + + public int getNumRecordsPerPage() { + return numRecordsPerPage; + } + + public void setFullPageRecords() { + numRecordsPerPage = 1; + bitmapSizeInBytes = 0; + heapFile.setEmptyPageMetadataSize((short) (heapFile.getEffectivePageSize() - + schema.getSizeInBytes())); + } + + public TableStats getStats() { + return stats; + } + + public long getNumRecords() { + return numRecords; + } + + public int getNumDataPages() { + return this.heapFile.getNumDataPages(); + } + + public int getPartNum() { + return heapFile.getPartNum(); + } + + private byte[] getBitMap(Page page) { + if (bitmapSizeInBytes > 0) { + byte[] bytes = new byte[bitmapSizeInBytes]; + page.getBuffer().get(bytes, 0, bitmapSizeInBytes); + return bytes; + } else { + return new byte[] {(byte) 0xFF}; + } + } + + private void writeBitMap(Page page, byte[] bitmap) { + if (bitmapSizeInBytes > 0) { + page.getBuffer().put(bitmap, 0, bitmapSizeInBytes); + } + } + + private static int computeBitmapSizeInBytes(int pageSize, Schema schema) { + int unroundedRecords = computeUnroundedNumRecordsPerPage(pageSize, schema); + if (unroundedRecords >= 8) { + // Dividing by 8 simultaneously (a) rounds down the number of records to a + // multiple of 8 and (b) converts bits to bytes. + return unroundedRecords / 8; + } else { + // special case: full page records with no bitmap + return 0; + } + } + + public static int computeNumRecordsPerPage(int pageSize, Schema schema) { + int unroundedRecords = computeUnroundedNumRecordsPerPage(pageSize, schema); + if (unroundedRecords >= 8) { + // Dividing by 8 and then multiplying by 8 rounds down to the nearest + // multiple of 8. + return computeUnroundedNumRecordsPerPage(pageSize, schema) / 8 * 8; + } else { + // special case: full page records with no bitmap + return 1; + } + } + + // Modifiers ///////////////////////////////////////////////////////////////// + /** + * buildStatistics builds histograms on each of the columns of a table. Running + * it multiple times refreshes the statistics + */ + public void buildStatistics(int buckets) { + this.stats.refreshHistograms(buckets, this); + } + + // Modifiers ///////////////////////////////////////////////////////////////// + private synchronized void insertRecord(Page page, int entryNum, Record record) { + int offset = bitmapSizeInBytes + (entryNum * schema.getSizeInBytes()); + page.getBuffer().position(offset).put(record.toBytes(schema)); + } + + /** + * addRecord adds a record to this table and returns the record id of the + * newly added record. stats, freePageNums, and numRecords are updated + * accordingly. The record is added to the first free slot of the first free + * page (if one exists, otherwise one is allocated). For example, if the + * first free page has bitmap 0b11101000, then the record is inserted into + * the page with index 3 and the bitmap is updated to 0b11111000. + */ + public synchronized RecordId addRecord(List values) { + Record record = schema.verify(values); + Page page = heapFile.getPageWithSpace(schema.getSizeInBytes()); + try { + // Find the first empty slot in the bitmap. + // entry number of the first free slot and store it in entryNum; and (2) we + // count the total number of entries on this page. + byte[] bitmap = getBitMap(page); + int entryNum = 0; + for (; entryNum < numRecordsPerPage; ++entryNum) { + if (Bits.getBit(bitmap, entryNum) == Bits.Bit.ZERO) { + break; + } + } + if (numRecordsPerPage == 1) { + entryNum = 0; + } + assert (entryNum < numRecordsPerPage); + + // Insert the record and update the bitmap. + insertRecord(page, entryNum, record); + Bits.setBit(bitmap, entryNum, Bits.Bit.ONE); + writeBitMap(page, bitmap); + + // Update the metadata. + stats.addRecord(record); + numRecords++; + + return new RecordId(page.getPageNum(), (short) entryNum); + } finally { + page.unpin(); + } + } + + /** + * Retrieves a record from the table, throwing an exception if no such record + * exists. + */ + public synchronized Record getRecord(RecordId rid) { + validateRecordId(rid); + Page page = fetchPage(rid.getPageNum()); + try { + byte[] bitmap = getBitMap(page); + if (Bits.getBit(bitmap, rid.getEntryNum()) == Bits.Bit.ZERO) { + String msg = String.format("Record %s does not exist.", rid); + throw new DatabaseException(msg); + } + + int offset = bitmapSizeInBytes + (rid.getEntryNum() * schema.getSizeInBytes()); + Buffer buf = page.getBuffer(); + buf.position(offset); + return Record.fromBytes(buf, schema); + } finally { + page.unpin(); + } + } + + /** + * Overwrites an existing record with new values and returns the existing + * record. stats is updated accordingly. An exception is thrown if rid does + * not correspond to an existing record in the table. + */ + public synchronized Record updateRecord(List values, RecordId rid) { + // TODO(hw4_part2): modify for smarter locking + + validateRecordId(rid); + + Record newRecord = schema.verify(values); + Record oldRecord = getRecord(rid); + + Page page = fetchPage(rid.getPageNum()); + try { + insertRecord(page, rid.getEntryNum(), newRecord); + + this.stats.removeRecord(oldRecord); + this.stats.addRecord(newRecord); + return oldRecord; + } finally { + page.unpin(); + } + } + + /** + * Deletes and returns the record specified by rid from the table and updates + * stats, freePageNums, and numRecords as necessary. An exception is thrown + * if rid does not correspond to an existing record in the table. + */ + public synchronized Record deleteRecord(RecordId rid) { + // TODO(hw4_part2): modify for smarter locking + + validateRecordId(rid); + + Page page = fetchPage(rid.getPageNum()); + try { + Record record = getRecord(rid); + + byte[] bitmap = getBitMap(page); + Bits.setBit(bitmap, rid.getEntryNum(), Bits.Bit.ZERO); + writeBitMap(page, bitmap); + + stats.removeRecord(record); + int numRecords = numRecordsPerPage == 1 ? 0 : numRecordsOnPage(page); + heapFile.updateFreeSpace(page, + (short) ((numRecordsPerPage - numRecords) * schema.getSizeInBytes())); + this.numRecords--; + + return record; + } finally { + page.unpin(); + } + } + + @Override + public String toString() { + return "Table " + name; + } + + // Helpers /////////////////////////////////////////////////////////////////// + private Page fetchPage(long pageNum) { + try { + return heapFile.getPage(pageNum); + } catch (PageException e) { + throw new DatabaseException(e); + } + } + + /** + * Recall that every data page contains an m-byte bitmap followed by n + * records. The following three functions computes m and n such that n is + * maximized. To simplify things, we round n down to the nearest multiple of + * 8 if necessary. m and n are stored in bitmapSizeInBytes and + * numRecordsPerPage respectively. + * + * Some examples: + * + * | Page Size | Record Size | bitmapSizeInBytes | numRecordsPerPage | + * | --------- | ----------- | ----------------- | ----------------- | + * | 9 bytes | 1 byte | 1 | 8 | + * | 10 bytes | 1 byte | 1 | 8 | + * ... + * | 17 bytes | 1 byte | 1 | 8 | + * | 18 bytes | 2 byte | 2 | 16 | + * | 19 bytes | 2 byte | 2 | 16 | + */ + private static int computeUnroundedNumRecordsPerPage(int pageSize, Schema schema) { + // Storing each record requires 1 bit for the bitmap and 8 * + // schema.getSizeInBytes() bits for the record. + int recordOverheadInBits = 1 + 8 * schema.getSizeInBytes(); + int pageSizeInBits = pageSize * 8; + return pageSizeInBits / recordOverheadInBits; + } + + private int numRecordsOnPage(Page page) { + byte[] bitmap = getBitMap(page); + int numRecords = 0; + for (int i = 0; i < numRecordsPerPage; ++i) { + if (Bits.getBit(bitmap, i) == Bits.Bit.ONE) { + numRecords++; + } + } + return numRecords; + } + + private void validateRecordId(RecordId rid) { + long p = rid.getPageNum(); + int e = rid.getEntryNum(); + + if (e < 0) { + String msg = String.format("Invalid negative entry number %d.", e); + throw new DatabaseException(msg); + } + + if (e >= numRecordsPerPage) { + String msg = String.format( + "There are only %d records per page, but record %d was requested.", + numRecordsPerPage, e); + throw new DatabaseException(msg); + } + } + + // Locking /////////////////////////////////////////////////////////////////// + + /** + * Enables auto-escalation. All future requests for pages of this table by transactions + * that hold locks on at least 20% of the locks on the table's pages when this table + * has at least 10 pages should escalate to a table-level lock before any locks are requested. + */ + public void enableAutoEscalate() { + // TODO(hw4_part2): implement + } + + /** + * Disables auto-escalation. No future requests for pages of this table should result in + * an automatic escalation to a table-level lock. + */ + public void disableAutoEscalate() { + // TODO(hw4_part2): implement + } + + // Iterators ///////////////////////////////////////////////////////////////// + public BacktrackingIterator ridIterator() { + // TODO(hw4_part2): reduce locking overhead for table scans + + BacktrackingIterator iter = heapFile.iterator(); + return new ConcatBacktrackingIterator<>(new PageIterator(iter, false)); + } + + @Override + public BacktrackingIterator iterator() { + return new RecordIterator(this, ridIterator()); + } + + private BacktrackingIterator blockRidIterator(Iterator pageIter, int maxPages) { + Page[] block = new Page[maxPages]; + int numPages; + for (numPages = 0; numPages < maxPages && pageIter.hasNext(); ++numPages) { + block[numPages] = pageIter.next(); + block[numPages].unpin(); + } + if (numPages < maxPages) { + Page[] temp = new Page[numPages]; + System.arraycopy(block, 0, temp, 0, numPages); + block = temp; + } + return new ConcatBacktrackingIterator<>(new PageIterator(new ArrayBacktrackingIterator<>(block), + true)); + } + + public BacktrackingIterator blockIterator(Iterator block, + int maxPages) { + return new RecordIterator(this, blockRidIterator(block, maxPages)); + } + + public BacktrackingIterator pageIterator() { + return heapFile.iterator(); + } + + /** + * RIDPageIterator is a BacktrackingIterator over the RecordIds of a single + * page of the table. + * + * See comments on the BacktrackingIterator interface for how mark and reset + * should function. + */ + class RIDPageIterator extends IndexBacktrackingIterator { + private Page page; + private byte[] bitmap; + + RIDPageIterator(Page page) { + super(numRecordsPerPage); + this.page = page; + this.bitmap = getBitMap(page); + page.unpin(); + } + + @Override + protected int getNextNonempty(int currentIndex) { + for (int i = currentIndex + 1; i < numRecordsPerPage; ++i) { + if (Bits.getBit(bitmap, i) == Bits.Bit.ONE) { + return i; + } + } + return numRecordsPerPage; + } + + @Override + protected RecordId getValue(int index) { + return new RecordId(page.getPageNum(), (short) index); + } + } + + private class PageIterator implements BacktrackingIterator> { + private BacktrackingIterator sourceIterator; + private boolean pinOnFetch; + + private PageIterator(BacktrackingIterator sourceIterator, boolean pinOnFetch) { + this.sourceIterator = sourceIterator; + this.pinOnFetch = pinOnFetch; + } + + @Override + public void markPrev() { + sourceIterator.markPrev(); + } + + @Override + public void markNext() { + sourceIterator.markNext(); + } + + @Override + public void reset() { + sourceIterator.reset(); + } + + @Override + public boolean hasNext() { + return sourceIterator.hasNext(); + } + + @Override + public BacktrackingIterable next() { + return new InnerIterable(sourceIterator.next()); + } + + private class InnerIterable implements BacktrackingIterable { + private Page baseObject; + + private InnerIterable(Page baseObject) { + this.baseObject = baseObject; + if (!pinOnFetch) { + baseObject.unpin(); + } + } + + @Override + public BacktrackingIterator iterator() { + baseObject.pin(); + return new RIDPageIterator(baseObject); + } + } + } +} + diff --git a/src/main/java/edu/berkeley/cs186/database/table/stats/Bucket.java b/src/main/java/edu/berkeley/cs186/database/table/stats/Bucket.java new file mode 100644 index 0000000..ed1ec2b --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/stats/Bucket.java @@ -0,0 +1,98 @@ +package edu.berkeley.cs186.database.table.stats; + +import java.util.Objects; +import java.util.HashSet; + +/** + * A histogram bucket. There are two types of buckets: + * + * 1. A bounded bucket `new Bucket(start, stop)` represents a count of + * values in the range [start, stop). + * 2. An unbounded bucket `new Bucket(start)` represents a count of values + * in the range [start, infinity). + */ +public class Bucket { + // If end is not null, then this bucket corresponds to range [start, stop). + // If end is null, then this bucket corresponds to the single value start. + private T start; + private T end; + private int count; + private int distinctCount; + + //todo fix later + private HashSet dictionary; + + public Bucket(T start) { + this(start, null); + } + + public Bucket(T start, T end) { + this.start = start; + this.end = end; + this.count = 0; + + this.distinctCount = 0; + this.dictionary = new HashSet<>(); + } + + public T getStart() { + return start; + } + + public T getEnd() { + return end; + } + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } + + public void setDistinctCount(int count) { + this.distinctCount = count; + dictionary = new HashSet<>(); + } + + public int getDistinctCount() { + return this.distinctCount + dictionary.size(); + } + + public void increment(float val) { + count ++; + dictionary.add(val); + } + + public void decrement(float val) { + count --; + dictionary.remove(val); + } + + @Override + public String toString() { + return String.format("[%s,%s):%d", start, end, count); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Bucket)) { + return false; + } + Bucket b = (Bucket) o; + boolean startEquals = start.equals(b.start); + boolean endEquals = (end == null && b.end == null) || + (end != null && b.end != null && end.equals(b.end)); + boolean countEquals = count == b.count; + return startEquals && endEquals && countEquals; + } + + @Override + public int hashCode() { + return Objects.hash(start, end, count); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/stats/Histogram.java b/src/main/java/edu/berkeley/cs186/database/table/stats/Histogram.java new file mode 100644 index 0000000..3c987a6 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/stats/Histogram.java @@ -0,0 +1,375 @@ +package edu.berkeley.cs186.database.table.stats; + +import java.util.Iterator; + +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.table.Table; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.databox.TypeId; + +/** + * A histogram maintains approximate statistics about a (potentially large) set + * of values without explicitly storing the values. For example, given the + * following set of numbers: + * + * 4, 9, 10, 10, 10, 13, 15, 16, 18, 18, 22, 23, 25, 26, 24, 42, 42, 42 + * + * We can build the following histogram: + * + * 10 | + * 9 | 8 + * 8 | +----+ + * 7 | | | + * 6 | | | + * 5 | | | 4 + * 4 | | +----+ 3 + * 3 | 2 | | | +----+ + * 2 | +----+ | | 1 | | + * 1 | | | | +----+ | + * 0 | | | | | | | + * ------------------------------ + * 0 10 20 30 40 50 + * + * A histogram is an ordered list of B "buckets", each of which defines a range (low, high). + * For the first, B - 1 buckets, the low of the range is inclusive an the high of the + * range is exclusive. For the last Bucket the high of the range is inclusive as well. + * Each bucket counts the number of values that fall within its range. In this project, + * you will work with a floating point histogram where low and high are defined by floats. + * For any other data type, we will map it so it fits into a floating point histogram. + * + * + * The primary data structure to consider is Bucket [] buckets, which is a list of Bucket + * objects + * + * Bucket b = new Bucket(10.0, 100.0); //defines a bucket whose low value is 10 and high is 100 + * b.getStart(); //returns 10.0 + * b.getEnd(); //returns 100.0 + * b.increment(15);// adds the value 15 to the bucket + * b.getCount();//returns the number of items added to the bucket + * b.getDistinctCount();//returns the approximate number of distinct items added to the bucket + * + * + */ +public class Histogram { + private Bucket[] buckets; //An array of float buckets the basic data structure + + private float minValue; + private float maxValue; + private float width; + + /*This constructor initialize an empty histogram object*/ + public Histogram() { + this(1); + } + + /*This constructor initialize a histogram object with a set number of buckets*/ + public Histogram(int numBuckets) { + buckets = new Bucket[numBuckets]; + for (int i = 0; i < numBuckets; ++i) { + buckets[i] = new Bucket<>(Float.MIN_VALUE, Float.MAX_VALUE); + } + } + + /*This is a copy constructor that generates a new histogram from a bucket list*/ + private Histogram(Bucket[] buckets) { + this.buckets = buckets; + this.minValue = buckets[0].getStart(); + this.width = buckets[0].getEnd() - buckets[0].getStart(); + this.maxValue = buckets[this.buckets.length - 1].getEnd(); + } + + /** We only consider float histograms, and these two methods turn every data type into a float. + * We call this mapping quantization. That means given any DataBox, we turn it into a float number. + * For Booleans, Integers, Floats, order is preserved in the mapping. But for strings, only equalities + * are preserved. + */ + private float quantization(Record record, int attribute) { + DataBox d = record.getValues().get(attribute); + return quantization(d); + } + + private float quantization(DataBox d) { + switch (d.type().getTypeId()) { + case BOOL: { return (d.getBool()) ? 1.0f : 0.0f; } + case INT: { return (float) d.getInt(); } + case FLOAT: { return d.getFloat(); } + case STRING: { return (float) (d.getString().hashCode()); } + } + + return 0f; + } + + /** buildHistogram() takes a table and an attribute and builds a fixed width histogram, with + * the following procedure. + * + * 1. Take a pass through the full table, and store the min and the max "quantized" value. + * 2. Calculate the width which is the (max - min)/#buckets + * 3. Create empty bucket objects and place them in the array. + * 4. Populate the buckets by incrementing + * + * Edge cases: width = 0, put an item only in the last bucket. + * final bucket is inclusive on the last value. + */ + public void buildHistogram(Table table, int attribute) { + // TODO(hw3_part2): implement + + //1. first calculate the min and the max values + + //2. calculate the width of each bin + + //3. create each bucket object + + //4. populate the data using the increment(value) method + + return; + } + + private int bucketIndex(float v) { + if (Math.abs(v - maxValue) < 0.00001) { return buckets.length - 1; } + return (int) Math.floor((v - minValue) / width); + } + + //Accessor Methods////////////////////////////////////////////////////////////// + /** Return an estimate of the number of distinct values in the histogram. */ + public int getNumDistinct() { + int sum = 0; + for (Bucket bucket : this.buckets) { + sum += bucket.getDistinctCount(); + } + + return sum; + } + + /** Return an estimate of the number of the total values in the histogram. */ + public int getCount() { + int sum = 0; + for (Bucket bucket : this.buckets) { + sum += bucket.getCount(); + } + + return sum; + } + + /* Returns the bucket object at i */ + public Bucket get(int i) { + return buckets[i]; + } + + //Operations////////////////////////////////////////////////////////////// + + /* Given a predicate, return a multiplicative mask for the histogram. That is, + * an array of size numBuckets where each entry is a float between 0 and 1 that represents a + * scaling to update the histogram count. Suppose we have this histogram with 5 buckets: + * + * 10 | + * 9 | 8 + * 8 | +----+ + * 7 | | | + * 6 | | | + * 5 | | | 4 + * 4 | | +----+ 3 + * 3 | 2 | | | +----+ + * 2 | +----+ | | 1 | | + * 1 | | | | +----+ | + * 0 | | | | | | | + * ------------------------------ + * 0 10 20 30 40 50 + * + * Then we get a mask, [0, .25, .5, 0, 1], the resulting histogram is: + * + * 10 | + * 9 | + * 8 | + * 7 | + * 6 | + * 5 | + * 4 | 3 + * 3 | 2 2 +----+ + * 2 | +----+----+ | | + * 1 | | | | | | + * 0 | 0 | | | 0 | | + * ------------------------------ + * 0 10 20 30 40 50 + * + * Counts are always an integer and round to the nearest value. + */ + public float[] filter(PredicateOperator predicate, DataBox value) { + float qvalue = quantization(value); + + //do not handle non equality predicates on strings + if (value.type().getTypeId() == TypeId.STRING && + predicate != PredicateOperator.EQUALS && + predicate != PredicateOperator.NOT_EQUALS) { + return stringNonEquality(qvalue); + } else if (predicate == PredicateOperator.EQUALS) { + return allEquality(qvalue); + } else if (predicate == PredicateOperator.NOT_EQUALS) { + return allNotEquality(qvalue); + } else if (predicate == PredicateOperator.GREATER_THAN) { + return allGreaterThan(qvalue); + } else if (predicate == PredicateOperator.LESS_THAN) { + return allLessThan(qvalue); + } else if (predicate == PredicateOperator.GREATER_THAN_EQUALS) { + return allGreaterThanEquals(qvalue); + } else { + return allLessThanEquals(qvalue); + } + } + + /** Given, we don't handle non equality comparisons of strings. Return 1*/ + private float [] stringNonEquality(float qvalue) { + float [] result = new float[this.buckets.length]; + for (int i = 0; i < this.buckets.length; i++) { + result[i] = 1.0f; + } + + return result; + } + + /*Nothing fancy here take max of gt and equals*/ + private float [] allGreaterThanEquals(float qvalue) { + float [] result = new float[this.buckets.length]; + float [] resultGT = allGreaterThan(qvalue); + float [] resultEquals = allEquality(qvalue); + + for (int i = 0; i < this.buckets.length; i++) { + result[i] = Math.max(resultGT[i], resultEquals[i]); + } + + return result; + } + + /*Nothing fancy here take max of lt and equals*/ + private float [] allLessThanEquals(float qvalue) { + float [] result = new float[this.buckets.length]; + float [] resultLT = allLessThan(qvalue); + float [] resultEquals = allEquality(qvalue); + + for (int i = 0; i < this.buckets.length; i++) { + result[i] = Math.max(resultLT[i], resultEquals[i]); + } + + return result; + } + + //Operations To Implement////////////////////////////////////////////////////////////// + + /** + * Given a quantized value, set the bucket that contains the value by 1/distinctCount, + * and set all other values to 0. + */ + private float [] allEquality(float qvalue) { + float [] result = new float[this.buckets.length]; + + // TODO(hw3_part2): implement + + return result; + } + + /** + * Given a quantized value, set the bucket that contains the value by 1-1/distinctCount, + * and set all other values to 1. + */ + private float [] allNotEquality(float qvalue) { + float [] result = new float[this.buckets.length]; + + // TODO(hw3_part2): implement + + return result; + } + + /** + * Given a quantized value, set the bucket that contains the value by (end - q)/width, + * and set all other buckets to 1 if higher and 0 if lower. + */ + private float [] allGreaterThan(float qvalue) { + float [] result = new float[this.buckets.length]; + + // TODO(hw3_part2): implement + + return result; + } + + /** + * Given a quantized value, set the bucket that contains the value by (q-start)/width, + * and set all other buckets to 1 if lower and 0 if higher. + */ + private float [] allLessThan(float qvalue) { + float [] result = new float[this.buckets.length]; + + // TODO(hw3_part2): implement + + return result; + } + + // Cost Estimation /////////////////////////////////////////////////////////////////// + + /** + * Return an estimate of the reduction factor for a given filter. For + * example, consider again the example histogram from the top of the file. + * The reduction factor for the predicate `>= 25` is 0.5 because roughly half + * of the values are greater than or equal to 25. + */ + public float computeReductionFactor(PredicateOperator predicate, DataBox value) { + float [] reduction = filter(predicate, value); + + float sum = 0.0f; + int total = 0; + + for (int i = 0; i < this.buckets.length; i++) { + //non empty buckets + sum += reduction[i] * this.buckets[i].getDistinctCount(); + total += this.buckets[i].getDistinctCount(); + } + + return sum / total; + } + + /** + * Given a histogram for a dataset, return a new histogram for the same + * dataset with a filter applied. For example, if apply the filter `>= 20` to + * the example histogram from the top of the file, we would get the following + * histogram: + * + * 6 | + * 5 | 4 + * 4 | +----+ 3 + * 3 | | | +----+ + * 2 | | | 1 | | + * 1 | 0 0 | +----+ | + * 0 | +----+----+ | | | + * ------------------------------ + * 0 1 2 3 4 5 + * 0 0 0 0 0 + */ + public Histogram copyWithPredicate(PredicateOperator predicate, DataBox value) { + float [] reduction = filter(predicate, value); + Bucket [] newBuckets = this.buckets.clone(); + + for (int i = 0; i < this.buckets.length; i++) { + int newCount = (int) Math.round(reduction[i] * this.buckets[i].getCount()); + int newDistinctCount = (int) Math.round(reduction[i] * this.buckets[i].getDistinctCount()); + + newBuckets[i].setCount(newCount); + newBuckets[i].setDistinctCount(newDistinctCount); + } + + return new Histogram(newBuckets); + } + + //uniformly reduces the values across the board with the mean reduction assumes uncorrelated + public Histogram copyWithReduction(float reduction) { + Bucket [] newBuckets = this.buckets.clone(); + + for (int i = 0; i < this.buckets.length; i++) { + int newCount = (int) Math.round(reduction * this.buckets[i].getCount()); + int newDistinctCount = (int) Math.round(reduction * this.buckets[i].getDistinctCount()); + + newBuckets[i].setCount(newCount); + newBuckets[i].setDistinctCount(newDistinctCount); + } + + return new Histogram(newBuckets); + } +} diff --git a/src/main/java/edu/berkeley/cs186/database/table/stats/TableStats.java b/src/main/java/edu/berkeley/cs186/database/table/stats/TableStats.java new file mode 100644 index 0000000..75a7351 --- /dev/null +++ b/src/main/java/edu/berkeley/cs186/database/table/stats/TableStats.java @@ -0,0 +1,249 @@ +package edu.berkeley.cs186.database.table.stats; + +import java.util.ArrayList; +import java.util.List; + +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.Table; + +/** + * Every table in a database maintains a set of table statistics which are + * updated whenever a tuple is added or deleted to it. These table statistics + * consist of an estimated number of records in the table, an estimated number + * of pages used by the table, and a histogram on every column of the table. + * For example, we can construct a TableStats and add/remove records from + * it like this: + * + * // Create a TableStats object for a table with columns (x: int, y: float). + * List fieldNames = Arrays.asList("x", "y"); + * List fieldTypes = Arrays.asList(Type.intType(), Type.floatType()); + * Schema schema = new Schema(fieldNames, fieldTypes); + * TableStats stats = new TableStats(schema); + * + * // Add and remove tuples from the stats. + * IntDataBox x1 = new IntDataBox(1); + * FloatDataBox y1 = new FloatDataBox(1); + * Record r1 = new Record(schema, Arrays.asList(x1, y1)); + * + * IntDataBox x2 = new IntDataBox(1); + * FloatDataBox y2 = new FloatDataBox(1); + * Record r2 = new Record(schema, Arrays.asList(x2, y2)); + * + * stats.addRecord(r1); + * stats.addRecord(r2); + * stats.removeRecord(r1); + * + * Later, we can use the statistics maintained by a TableStats object for + * things like query optimization: + * + * stats.getNumRecords(); // Estimated number of records. + * stats.getNumPages(); // Estimated number of pages. + * stats.getHistograms(); // Histograms on each column. + */ +public class TableStats { + private Schema tableSchema; + private int numRecordsPerPage; + private int numRecords; + private List histograms; + + /** Construct a TableStats for an empty table with schema `tableSchema`. */ + public TableStats(Schema tableSchema, int numRecordsPerPage) { + this.tableSchema = tableSchema; + this.numRecordsPerPage = numRecordsPerPage; + this.numRecords = 0; + this.histograms = new ArrayList<>(); + for (Type t : tableSchema.getFieldTypes()) { + Histogram h = new Histogram(); + this.histograms.add(h); + } + } + + private TableStats(Schema tableSchema, int numRecordsPerPage, int numRecords, + List histograms) { + this.tableSchema = tableSchema; + this.numRecordsPerPage = numRecordsPerPage; + this.numRecords = numRecords; + this.histograms = histograms; + } + + // Modifiers ///////////////////////////////////////////////////////////////// + public void addRecord(Record record) { + numRecords++; + } + + public void refreshHistograms(int buckets, Table tab) { + List newHistograms = new ArrayList<>(); + int count = 0; + int totalRecords = 0; + for (Type t : tableSchema.getFieldTypes()) { + Histogram h = new Histogram(buckets); + h.buildHistogram(tab, count); + newHistograms.add(h); + totalRecords += h.getCount(); + count++; + } + + this.histograms = newHistograms; + this.numRecords = Math.round(((float)totalRecords) / count); + } + + public void removeRecord(Record record) { + numRecords = Math.max(numRecords - 1, 0); + } + + // Accessors ///////////////////////////////////////////////////////////////// + public Schema getSchema() { + return tableSchema; + } + + public int getNumRecords() { + return numRecords; + } + + /** + * Calculates the number of data pages required to store `numRecords` records + * assuming that all records are stored as densely as possible in the pages. + */ + public int getNumPages() { + if (numRecords % numRecordsPerPage == 0) { + return numRecords / numRecordsPerPage; + } else { + return (numRecords / numRecordsPerPage) + 1; + } + } + + public List getHistograms() { + return histograms; + } + + // Copiers /////////////////////////////////////////////////////////////////// + /** + * Estimates the table statistics for the table that would be produced after + * filtering column `i` with `predicate` and `value`. For simplicity, we + * assume that columns are completeley uncorrelated. For example, imagine the + * following table statistics for a table T(x:int, y:int). + * + * numRecords = 100 + * numPages = 2 + * Histogram x Histogram y + * =========== =========== + * 60 | 50 60 | + * 50 | 40 +----+ 50 | + * 40 | +----+ | | 40 | + * 30 | | | | | 30 | 20 20 20 20 20 + * 20 | 10 | | | | 20 | +----+----+----+----+----+ + * 10 | +----+ | 00 00 | | 10 | | | | | | | + * 00 | | | +----+----+ | 00 | | | | | | | + * ---------------------------- ---------------------------- + * 0 1 2 3 4 5 0 1 2 3 4 5 + * 0 0 0 0 0 0 0 0 0 0 + * + * If we apply the filter `x < 20`, we estimate that we would have the + * following table statistics. + * + * numRecords = 50 + * numPages = 1 + * Histogram x Histogram y + * =========== =========== + * 50 | 40 50 | + * 40 | +----+ 40 | + * 30 | | | 30 | + * 20 | 10 | | 20 | 10 10 10 10 10 + * 10 | +----+ | 00 00 50 10 | +----+----+----+----+----+ + * 00 | | | +----+----+----+ 00 | | | | | | | + * ---------------------------- ---------------------------- + * 0 1 2 3 4 5 0 1 2 3 4 5 + * 0 0 0 0 0 0 0 0 0 0 + */ + public TableStats copyWithPredicate(int column, + PredicateOperator predicate, + DataBox d) { + float reductionFactor = histograms.get(column).computeReductionFactor(predicate, d); + List copyHistograms = new ArrayList<>(); + for (int j = 0; j < histograms.size(); ++j) { + Histogram histogram = histograms.get(j); + if (column == j) { + copyHistograms.add(histogram.copyWithPredicate(predicate, d)); + } else { + copyHistograms.add(histogram.copyWithReduction(reductionFactor)); + } + } + + Histogram qhistogram = histograms.get(column); + int numRecords = qhistogram.getCount(); + return new TableStats(this.tableSchema, this.numRecordsPerPage, numRecords, copyHistograms); + } + + /** + * Creates a new TableStats which is the statistics for the table + * that results from this TableStats joined with the given TableStats. + * + * @param leftIndex the index of the join column for this + * @param rightStats the TableStats of the right table to be joined + * @param rightIndex the index of the join column for the right table + * @return new TableStats based off of this and params + */ + public TableStats copyWithJoin(int leftIndex, + TableStats rightStats, + int rightIndex) { + // Compute the new schema. + List joinedFieldNames = new ArrayList<>(); + joinedFieldNames.addAll(tableSchema.getFieldNames()); + joinedFieldNames.addAll(rightStats.tableSchema.getFieldNames()); + + List joinedFieldTypes = new ArrayList<>(); + joinedFieldTypes.addAll(tableSchema.getFieldTypes()); + joinedFieldTypes.addAll(rightStats.tableSchema.getFieldTypes()); + + Schema joinedSchema = new Schema(joinedFieldNames, joinedFieldTypes); + + //System.out.println(this.numRecords + " " +rightStats.getNumRecords()); + + int inputSize = this.numRecords * rightStats.getNumRecords(); + + int leftNumDistinct; + if (this.histograms.size() > 0) { + leftNumDistinct = this.histograms.get(leftIndex).getNumDistinct() + 1; + } else { + leftNumDistinct = 1; + } + + int rightNumDistinct; + if (rightStats.histograms.size() > 0) { + rightNumDistinct = rightStats.histograms.get(rightIndex).getNumDistinct() + 1; + } else { + rightNumDistinct = 1; + } + + float reductionFactor = 1.0f / Math.max(leftNumDistinct, rightNumDistinct); + + List copyHistograms = new ArrayList<>(); + + int leftNumRecords = this.numRecords; + int rightNumRecords = rightStats.getNumRecords(); + + //todo fix + float leftReductionFactor = leftNumDistinct / Math.max(leftNumDistinct, rightNumDistinct); + float rightReductionFactor = rightNumDistinct / Math.max(leftNumDistinct, rightNumDistinct); + + float joinReductionFactor = leftReductionFactor; + +// Histogram joinHistogram = this.histograms.get(leftIndex); + + for (Histogram leftHistogram : this.histograms) { + copyHistograms.add(leftHistogram.copyWithReduction(leftReductionFactor)); + } + + for (Histogram rightHistogram : rightStats.histograms) { + copyHistograms.add(rightHistogram.copyWithReduction(rightReductionFactor)); + } + + int outputSize = (int)(reductionFactor * inputSize); + + return new TableStats(joinedSchema, this.numRecordsPerPage, outputSize, copyHistograms); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/TestDatabase.java b/src/test/java/edu/berkeley/cs186/database/TestDatabase.java new file mode 100644 index 0000000..17c9fff --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/TestDatabase.java @@ -0,0 +1,413 @@ +package edu.berkeley.cs186.database; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.databox.*; +import edu.berkeley.cs186.database.query.QueryPlan; +import edu.berkeley.cs186.database.table.*; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.Rule; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.*; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestDatabase { + private static final String TestDir = "testDatabase"; + private Database db; + private String filename; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void beforeEach() throws Exception { + File testDir = tempFolder.newFolder(TestDir); + this.filename = testDir.getAbsolutePath(); + this.db = new Database(filename, 32); + this.db.setWorkMem(4); + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + } + } + + @After + public void afterEach() { + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + } + this.db.close(); + } + + @Test + public void testTableCreate() { + Schema s = TestUtils.createSchemaWithAllTypes(); + + try(Transaction t = this.db.beginTransaction()) { + t.createTable(s, "testTable1"); + } + } + + @Test + public void testTransactionBegin() { + Schema s = TestUtils.createSchemaWithAllTypes(); + Record input = TestUtils.createRecordWithAllTypes(); + + String tableName = "testTable1"; + + try(Transaction t1 = db.beginTransaction()) { + t1.createTable(s, tableName); + RecordId rid = t1.getTransactionContext().addRecord(tableName, input.getValues()); + t1.getTransactionContext().getRecord(tableName, rid); + } + } + + @Test + public void testTransactionTempTable() { + Schema s = TestUtils.createSchemaWithAllTypes(); + Record input = TestUtils.createRecordWithAllTypes(); + + String tableName = "testTable1"; + + try(Transaction t1 = db.beginTransaction()) { + t1.createTable(s, tableName); + RecordId rid = t1.getTransactionContext().addRecord(tableName, input.getValues()); + Record rec = t1.getTransactionContext().getRecord(tableName, rid); + assertEquals(input, rec); + + String tempTableName = t1.getTransactionContext().createTempTable(s); + rid = t1.getTransactionContext().addRecord(tempTableName, input.getValues()); + rec = t1.getTransactionContext().getRecord(tempTableName, rid); + assertEquals(input, rec); + } + } + + @Test(expected = DatabaseException.class) + public void testTransactionTempTable2() { + Schema s = TestUtils.createSchemaWithAllTypes(); + Record input = TestUtils.createRecordWithAllTypes(); + + String tableName = "testTable1"; + + RecordId rid; + Record rec; + String tempTableName; + try(Transaction t1 = db.beginTransaction()) { + t1.createTable(s, tableName); + rid = t1.getTransactionContext().addRecord(tableName, input.getValues()); + rec = t1.getTransactionContext().getRecord(tableName, rid); + assertEquals(input, rec); + + tempTableName = t1.getTransactionContext().createTempTable(s); + rid = t1.getTransactionContext().addRecord(tempTableName, input.getValues()); + rec = t1.getTransactionContext().getRecord(tempTableName, rid); + assertEquals(input, rec); + } + + try(Transaction t2 = db.beginTransaction()) { + t2.getTransactionContext().addRecord(tempTableName, input.getValues()); + } + } + + @Test + public void testDatabaseDurablity() { + Schema s = TestUtils.createSchemaWithAllTypes(); + Record input = TestUtils.createRecordWithAllTypes(); + + String tableName = "testTable1"; + + RecordId rid; + Record rec; + try(Transaction t1 = db.beginTransaction()) { + t1.createTable(s, tableName); + rid = t1.getTransactionContext().addRecord(tableName, input.getValues()); + rec = t1.getTransactionContext().getRecord(tableName, rid); + + assertEquals(input, rec); + } + + db.close(); + db = new Database(this.filename, 32); + + try(Transaction t1 = db.beginTransaction()) { + rec = t1.getTransactionContext().getRecord(tableName, rid); + assertEquals(input, rec); + } + } + + @Test + public void testREADMESample() { + try (Transaction t1 = db.beginTransaction()) { + Schema s = new Schema( + Arrays.asList("id", "firstName", "lastName"), + Arrays.asList(Type.intType(), Type.stringType(10), Type.stringType(10)) + ); + t1.createTable(s, "table1"); + t1.insert("table1", Arrays.asList( + new IntDataBox(1), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + )); + t1.insert("table1", Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + )); + t1.commit(); + } + + try (Transaction t2 = db.beginTransaction()) { + Iterator iter = t2.query("table1").execute(); + + assertEquals(Arrays.asList( + new IntDataBox(1), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + ), iter.next().getValues()); + + assertEquals(Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + ), iter.next().getValues()); + + assertFalse(iter.hasNext()); + + t2.commit(); + } + } + + @Test + public void testJoinQuery() { + try (Transaction t1 = db.beginTransaction()) { + Schema s = new Schema( + Arrays.asList("id", "firstName", "lastName"), + Arrays.asList(Type.intType(), Type.stringType(10), Type.stringType(10)) + ); + t1.createTable(s, "table1"); + t1.insert("table1", Arrays.asList( + new IntDataBox(1), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + )); + t1.insert("table1", Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + )); + t1.commit(); + } + + try (Transaction t2 = db.beginTransaction()) { + // FROM table1 AS t1 + QueryPlan queryPlan = t2.query("table1", "t1"); + // JOIN table1 AS t2 ON t1.lastName = t2.lastName + queryPlan.join("table1", "t2", "t1.lastName", "t2.lastName"); + // WHERE t1.firstName = 'John' + queryPlan.select("t1.firstName", PredicateOperator.EQUALS, new StringDataBox("John", 10)); + // .. AND t2.firstName = 'Jane' + queryPlan.select("t2.firstName", PredicateOperator.EQUALS, new StringDataBox("Jane", 10)); + // SELECT t1.id, t2.id, t1.firstName, t2.firstName, t1.lastName + queryPlan.project(Arrays.asList("t1.id", "t2.id", "t1.firstName", "t2.firstName", "t1.lastName")); + + // run the query + Iterator iter = queryPlan.execute(); + + assertEquals(Arrays.asList( + new IntDataBox(1), + new IntDataBox(2), + new StringDataBox("John", 10), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + ), iter.next().getValues()); + + assertFalse(iter.hasNext()); + + t2.commit(); + } + } + + @Test + public void testAggQuery() { + try (Transaction t1 = db.beginTransaction()) { + Schema s = new Schema( + Arrays.asList("id", "firstName", "lastName"), + Arrays.asList(Type.intType(), Type.stringType(10), Type.stringType(10)) + ); + t1.createTable(s, "table1"); + t1.insert("table1", Arrays.asList( + new IntDataBox(1), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + )); + t1.insert("table1", Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + )); + t1.commit(); + } + + try (Transaction t2 = db.beginTransaction()) { + // FROM table1 + QueryPlan queryPlan = t2.query("table1"); + // SELECT COUNT(*) + queryPlan.count(); + // .. SUM(id) + queryPlan.sum("id"); + // .. AVERAGE(id) + queryPlan.average("id"); + + // run the query + Iterator iter = queryPlan.execute(); + + assertEquals(Arrays.asList( + new IntDataBox(2), + new IntDataBox(3), + new FloatDataBox(1.5f) + ), iter.next().getValues()); + + assertFalse(iter.hasNext()); + + t2.commit(); + } + } + + @Test + public void testGroupByQuery() { + try (Transaction t1 = db.beginTransaction()) { + Schema s = new Schema( + Arrays.asList("id", "firstName", "lastName"), + Arrays.asList(Type.intType(), Type.stringType(10), Type.stringType(10)) + ); + t1.createTable(s, "table1"); + t1.insert("table1", Arrays.asList( + new IntDataBox(1), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + )); + t1.insert("table1", Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + )); + + t1.commit(); + } + + try (Transaction t2 = db.beginTransaction()) { + // FROM table1 + QueryPlan queryPlan = t2.query("table1"); + // GROUP BY lastName + queryPlan.groupBy("lastName"); + // SELECT lastName + queryPlan.project(Collections.singletonList("lastName")); + // .. COUNT(*) + queryPlan.count(); + + // run the query + Iterator iter = queryPlan.execute(); + + assertEquals(Arrays.asList( + new StringDataBox("Doe", 10), + new IntDataBox(2) + ), iter.next().getValues()); + + assertFalse(iter.hasNext()); + + t2.commit(); + } + } + + @Test + public void testUpdateQuery() { + try (Transaction t1 = db.beginTransaction()) { + Schema s = new Schema( + Arrays.asList("id", "firstName", "lastName"), + Arrays.asList(Type.intType(), Type.stringType(10), Type.stringType(10)) + ); + t1.createTable(s, "table1"); + t1.insert("table1", Arrays.asList( + new IntDataBox(1), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + )); + t1.insert("table1", Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + )); + + t1.commit(); + } + + try (Transaction t2 = db.beginTransaction()) { + // UPDATE table1 SET id = id + 10 WHERE lastName = 'Doe' + t2.update("table1", "id", (DataBox x) -> new IntDataBox(x.getInt() + 10), + "lastName", PredicateOperator.EQUALS, new StringDataBox("Doe", 10)); + + Iterator iter = t2.query("table1").execute(); + + assertEquals(Arrays.asList( + new IntDataBox(11), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + ), iter.next().getValues()); + + assertEquals(Arrays.asList( + new IntDataBox(12), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + ), iter.next().getValues()); + + assertFalse(iter.hasNext()); + } + } + + @Test + public void testDeleteQuery() { + try (Transaction t1 = db.beginTransaction()) { + Schema s = new Schema( + Arrays.asList("id", "firstName", "lastName"), + Arrays.asList(Type.intType(), Type.stringType(10), Type.stringType(10)) + ); + t1.createTable(s, "table1"); + t1.insert("table1", Arrays.asList( + new IntDataBox(1), + new StringDataBox("John", 10), + new StringDataBox("Doe", 10) + )); + t1.insert("table1", Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + )); + + t1.commit(); + } + + try (Transaction t2 = db.beginTransaction()) { + // DELETE FROM table1 WHERE id <> 2 + t2.delete("table1", "id", PredicateOperator.NOT_EQUALS, new IntDataBox(2)); + + Iterator iter = t2.query("table1").execute(); + + assertEquals(Arrays.asList( + new IntDataBox(2), + new StringDataBox("Jane", 10), + new StringDataBox("Doe", 10) + ), iter.next().getValues()); + + assertFalse(iter.hasNext()); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/TestDatabaseDeadlockPrecheck.java b/src/test/java/edu/berkeley/cs186/database/TestDatabaseDeadlockPrecheck.java new file mode 100644 index 0000000..7590499 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/TestDatabaseDeadlockPrecheck.java @@ -0,0 +1,82 @@ +package edu.berkeley.cs186.database; + +import edu.berkeley.cs186.database.categories.HW4Part2Tests; +import edu.berkeley.cs186.database.categories.HW4Tests; +import edu.berkeley.cs186.database.categories.PublicTests; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.LockType; +import edu.berkeley.cs186.database.concurrency.LoggingLockManager; +import edu.berkeley.cs186.database.concurrency.ResourceName; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.*; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; + +import static org.junit.Assert.assertTrue; + +@Category({HW4Tests.class, HW4Part2Tests.class}) +public class TestDatabaseDeadlockPrecheck { + private static final String TestDir = "testDatabaseDeadlockPrecheck"; + + // 7 second max per method tested. + public static long timeout = (long) (7000 * TimeoutScaling.factor); + + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis(timeout)); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + @Category(PublicTests.class) + public void testDeadlock() { + assertTrue(performCheck(tempFolder)); + } + + public static boolean performCheck(TemporaryFolder checkFolder) { + // If we are unable to request an X lock after an X lock is requested and released, there is no point + // running any of the tests in this class - every test will block the main thread. + final ResourceName name = new ResourceName(new Pair<>("database", 0L)); + final LockType lockType = LockType.X; + + Thread mainRunner = new Thread(() -> { + try { + File testDir = checkFolder.newFolder(TestDir); + String filename = testDir.getAbsolutePath(); + LoggingLockManager lockManager = new LoggingLockManager(); + Database database = new Database(filename, 128, lockManager); + database.setWorkMem(32); + database.waitSetupFinished(); + try(Transaction transaction = database.beginTransaction()) { + lockManager.acquire(transaction.getTransactionContext(), name, lockType); + } + try(Transaction transaction = database.beginTransaction()) { + lockManager.acquire(transaction.getTransactionContext(), name, lockType); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + mainRunner.start(); + try { + if ((new DisableOnDebug(new TestName()).isDebugging())) { + mainRunner.join(); + } else { + mainRunner.join(timeout); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + return mainRunner.getState() == Thread.State.TERMINATED; + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/TestDatabaseLocking.java b/src/test/java/edu/berkeley/cs186/database/TestDatabaseLocking.java new file mode 100644 index 0000000..9648616 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/TestDatabaseLocking.java @@ -0,0 +1,806 @@ +package edu.berkeley.cs186.database; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.concurrency.*; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.query.QueryPlan; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.RecordId; +import edu.berkeley.cs186.database.table.Schema; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.*; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.*; +import java.util.concurrent.Phaser; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +@Category({HW4Tests.class, HW4Part2Tests.class}) +public class TestDatabaseLocking { + private static final String TestDir = "testDatabaseLocking"; + private static boolean passedPreCheck = false; + private Database db; + private LoggingLockManager lockManager; + private String filename; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + // 7 second max per method tested. + public static long timeout = (long) (7000 * TimeoutScaling.factor); + + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis(timeout)); + + private void reloadDatabase() { + if (this.db != null) { + while (TransactionContext.getTransaction() != null) { + TransactionContext.unsetTransaction(); + } + this.db.close(); + } + if (this.lockManager != null && this.lockManager.isLogging()) { + List oldLog = this.lockManager.log; + this.lockManager = new LoggingLockManager(); + this.lockManager.log = oldLog; + this.lockManager.startLog(); + } else { + this.lockManager = new LoggingLockManager(); + } + this.db = new Database(this.filename, 128, this.lockManager); + this.db.setWorkMem(32); // B=32 + // force initialization to finish before continuing + this.db.waitSetupFinished(); + this.db.waitAllTransactions(); + } + + @ClassRule + public static TemporaryFolder checkFolder = new TemporaryFolder(); + + @BeforeClass + public static void beforeAll() { + passedPreCheck = TestDatabaseDeadlockPrecheck.performCheck(checkFolder); + } + + @Before + public void beforeEach() throws Exception { + assertTrue(passedPreCheck); + + File testDir = tempFolder.newFolder(TestDir); + this.filename = testDir.getAbsolutePath(); + this.reloadDatabase(); + try(Transaction t = this.beginTransaction()) { + t.dropAllTables(); + } finally { + this.db.waitAllTransactions(); + } + } + + @After + public void afterEach() { + if (!passedPreCheck) { + return; + } + + this.lockManager.endLog(); + while (TransactionContext.getTransaction() != null) { + TransactionContext.unsetTransaction(); + } + this.db.close(); + } + + private Transaction beginTransaction() { + // Database.Transaction ordinarily calls setTransaction/unsetTransaction around calls, + // but we test directly with TransactionContext calls here, so we need to call setTransaction + // manually + Transaction t = db.beginTransaction(); + TransactionContext.setTransaction(t.getTransactionContext()); + return t; + } + + private static > void assertSameItems(List expected, + List actual) { + Collections.sort(expected); + Collections.sort(actual); + assertEquals(expected, actual); + } + + private static void assertSubsequence(List expected, List actual) { + if (expected.size() == 0) { + return; + } + Iterator ei = expected.iterator(); + Iterator ai = actual.iterator(); + while (ei.hasNext()) { + T next = ei.next(); + boolean found = false; + while (ai.hasNext()) { + if (ai.next().equals(next)) { + found = true; + break; + } + } + assertTrue(expected + " not subsequence of " + actual, found); + } + } + + private static void assertContainsAll(List expected, List actual) { + if (expected.size() == 0) { + return; + } + for (T item : expected) { + assertTrue(item + " not in " + actual, actual.contains(item)); + } + } + + private static List prepare(Long transNum, String ... expected) { + return Arrays.stream(expected).map((String log) -> String.format(log, + transNum)).collect(Collectors.toList()); + } + + private static List removeMetadataLogs(List log) { + log = new ArrayList<>(log); + // remove all information_schema lock log entries + log.removeIf((String x) -> x.contains("information_schema")); + // replace [acquire IS(database), promote IX(database), ...] with [acquire IX(database), ...] + // (as if the information_schema locks never happened) + if (log.size() >= 2 && log.get(0).endsWith("database IS") && log.get(1).endsWith("database IX")) { + log.set(0, log.get(0).replace("IS", "IX")); + log.remove(1); + } + return log; + } + + private List createTable(String tableName, int pages) { + Schema s = TestUtils.createSchemaWithAllTypes(); + Record input = TestUtils.createRecordWithAllTypes(); + List values = input.getValues(); + List rids = new ArrayList<>(); + try(Transaction t1 = beginTransaction()) { + t1.createTable(s, tableName); + int numRecords = pages * t1.getTransactionContext().getTable(tableName).getNumRecordsPerPage(); + for (int i = 0; i < numRecords; ++i) { + rids.add(t1.getTransactionContext().addRecord(tableName, values)); + } + } finally { + this.db.waitAllTransactions(); + } + + return rids; + } + + private List createTableWithIndices(String tableName, int pages, + List indexColumns) { + Schema s = TestUtils.createSchemaWithTwoInts(); + List rids = new ArrayList<>(); + try(Transaction t1 = beginTransaction()) { + t1.createTable(s, tableName); + for (String col : indexColumns) { + t1.createIndex(tableName, col, false); + } + int numRecords = pages * t1.getTransactionContext().getTable(tableName).getNumRecordsPerPage(); + for (int i = 0; i < numRecords; ++i) { + rids.add(t1.getTransactionContext().addRecord(tableName, Arrays.asList(new IntDataBox(i), + new IntDataBox(i)))); + } + } finally { + this.db.waitAllTransactions(); + } + + return rids; + } + + @Test + @Category(PublicTests.class) + public void testRecordRead() { + String tableName = "testTable1"; + List rids = createTable(tableName, 4); + lockManager.startLog(); + + try(Transaction t1 = beginTransaction()) { + t1.getTransactionContext().getRecord(tableName, rids.get(0)); + t1.getTransactionContext().getRecord(tableName, rids.get(3 * rids.size() / 4 - 1)); + t1.getTransactionContext().getRecord(tableName, rids.get(rids.size() - 1)); + + assertEquals(prepare(t1.getTransNum(), + "acquire %s database IS", + "acquire %s database/tables.testTable1 IS", + "acquire %s database/tables.testTable1/30000000001 S", + "acquire %s database/tables.testTable1/30000000003 S", + "acquire %s database/tables.testTable1/30000000004 S" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testSimpleTransactionCleanup() { + String tableName = "testTable1"; + List rids = createTable(tableName, 4); + + Transaction t1 = beginTransaction(); + try { + t1.getTransactionContext().getRecord(tableName, rids.get(0)); + t1.getTransactionContext().getRecord(tableName, rids.get(3 * rids.size() / 4 - 1)); + t1.getTransactionContext().getRecord(tableName, rids.get(rids.size() - 1)); + + assertTrue("did not acquire all required locks", + lockManager.getLocks(t1.getTransactionContext()).size() >= 5); + + lockManager.startLog(); + } finally { + t1.commit(); + this.db.waitAllTransactions(); + } + + assertTrue("did not free all required locks", + lockManager.getLocks(t1.getTransactionContext()).isEmpty()); + assertSubsequence(prepare(t1.getTransNum(), + "release %s database/tables.testTable1/30000000003", + "release %s database" + ), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testRecordWrite() { + String tableName = "testTable1"; + List rids = createTable(tableName, 4); + Record input = TestUtils.createRecordWithAllTypes(); + List values = input.getValues(); + + try(Transaction t0 = beginTransaction()) { + t0.getTransactionContext().deleteRecord(tableName, rids.get(rids.size() - 1)); + } finally { + this.db.waitAllTransactions(); + } + + lockManager.startLog(); + + try(Transaction t1 = beginTransaction()) { + t1.getTransactionContext().addRecord(tableName, values); + + assertEquals(prepare(t1.getTransNum(), + "acquire %s database IX", + "acquire %s database/tables.testTable1 IX", + "acquire %s database/tables.testTable1/30000000004 X" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testRecordUpdate() { + String tableName = "testTable1"; + List rids = createTable(tableName, 4); + Record input = TestUtils.createRecordWithAllTypes(); + List values = input.getValues(); + + lockManager.startLog(); + + try(Transaction t1 = beginTransaction()) { + t1.getTransactionContext().updateRecord(tableName, values, rids.get(rids.size() - 1)); + + assertEquals(prepare(t1.getTransNum(), + "acquire %s database IX", + "acquire %s database/tables.testTable1 IX", + "acquire %s database/tables.testTable1/30000000004 X" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testRecordDelete() { + String tableName = "testTable1"; + List rids = createTable(tableName, 4); + + lockManager.startLog(); + + try(Transaction t1 = beginTransaction()) { + t1.getTransactionContext().deleteRecord(tableName, rids.get(rids.size() - 1)); + + assertEquals(prepare(t1.getTransNum(), + "acquire %s database IX", + "acquire %s database/tables.testTable1 IX", + "acquire %s database/tables.testTable1/30000000004 X" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testTableScan() { + String tableName = "testTable1"; + createTable(tableName, 4); + + lockManager.startLog(); + + try(Transaction t1 = beginTransaction()) { + Iterator r = t1.getTransactionContext().getRecordIterator(tableName); + while (r.hasNext()) { + r.next(); + } + + assertEquals(prepare(t1.getTransNum(), + "acquire %s database IS", + "acquire %s database/tables.testTable1 S" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testSortedScanNoIndexLocking() { + String tableName = "testTable1"; + createTable(tableName, 1); + + lockManager.startLog(); + + try(Transaction t1 = beginTransaction()) { + Iterator r = t1.getTransactionContext().sortedScan(tableName, "int"); + while (r.hasNext()) { + r.next(); + } + + assertEquals(prepare(t1.getTransNum(), + "acquire %s database IS", + "acquire %s database/tables.testTable1 S" + ), removeMetadataLogs(lockManager.log).subList(0, 2)); + } + } + + @Test + @Category(PublicTests.class) + public void testBPlusTreeRestrict() { + String tableName = "testTable1"; + lockManager.startLog(); + createTableWithIndices(tableName, 0, Collections.singletonList("int1")); + assertTrue(lockManager.log.contains("disable-children database/indices.testTable1,int1")); + } + + @Test + @Category(PublicTests.class) + public void testSortedScanLocking() { + String tableName = "testTable1"; + List rids = createTableWithIndices(tableName, 1, Arrays.asList("int1", "int2")); + + lockManager.startLog(); + try(Transaction t1 = beginTransaction()) { + Iterator r = t1.getTransactionContext().sortedScan(tableName, "int1"); + while (r.hasNext()) { + r.next(); + } + List log = removeMetadataLogs(lockManager.log); + assertEquals(3, log.size()); + assertEquals(prepare(t1.getTransNum(), + "acquire %s database IS", + "acquire %s database/tables.testTable1 S", + "acquire %s database/indices.testTable1,int1 S" + ), log); + } finally { + this.db.waitAllTransactions(); + } + + lockManager.clearLog(); + try(Transaction t2 = beginTransaction()) { + Iterator r = t2.getTransactionContext().sortedScanFrom(tableName, "int2", + new IntDataBox(rids.size() / 2)); + while (r.hasNext()) { + r.next(); + } + List log = removeMetadataLogs(lockManager.log); + assertEquals(3, log.size()); + assertEquals(prepare(t2.getTransNum(), + "acquire %s database IS", + "acquire %s database/tables.testTable1 S", + "acquire %s database/indices.testTable1,int2 S" + ), log); + } + } + + @Test + @Category(PublicTests.class) + public void testSearchOperationLocking() { + String tableName = "testTable1"; + List rids = createTableWithIndices(tableName, 1, Arrays.asList("int1", "int2")); + + lockManager.startLog(); + try(Transaction t1 = beginTransaction()) { + t1.getTransactionContext().lookupKey(tableName, "int1", new IntDataBox(rids.size() / 2)); + assertEquals(prepare(t1.getTransNum(), + "acquire %s database IS", + "acquire %s database/indices.testTable1,int1 S" + ), removeMetadataLogs(lockManager.log)); + } finally { + this.db.waitAllTransactions(); + } + + lockManager.clearLog(); + try(Transaction t2 = beginTransaction()) { + t2.getTransactionContext().contains(tableName, "int2", new IntDataBox(rids.size() / 2 - 1)); + assertEquals(prepare(t2.getTransNum(), + "acquire %s database IS", + "acquire %s database/indices.testTable1,int2 S" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testQueryWithIndex() { + String tableName = "testTable1"; + createTableWithIndices(tableName, 6, Arrays.asList("int1", "int2")); + + try (Transaction ts = beginTransaction()) { + ts.getTransactionContext().getTable("testTable1").buildStatistics(10); + } + + db.waitAllTransactions(); + lockManager.startLog(); + + try(Transaction t0 = beginTransaction()) { + QueryPlan q = t0.query(tableName); + q.select("int1", PredicateOperator.EQUALS, new IntDataBox(2)); + q.project(Collections.singletonList("int2")); + q.execute(); + + assertEquals(prepare(t0.getTransNum(), + "acquire %s database IS", + "acquire %s database/indices.testTable1,int1 S" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testPageDirectoryCapacityLoad() { + String tableName = "testTable1"; + createTable(tableName, 0); + + while (TransactionContext.getTransaction() != null) { + TransactionContext.unsetTransaction(); + } + db.close(); + db = null; + + this.lockManager = new LoggingLockManager(); + lockManager.startLog(); + + this.reloadDatabase(); + assertTrue(lockManager.log.contains("set-capacity database/tables.testTable1 0")); + } + + @Test + @Category(PublicTests.class) + public void testAutoEscalateS() { + String tableName = "testTable1"; + List rids = createTable(tableName, 18); + + lockManager.startLog(); + + try(Transaction t0 = beginTransaction()) { + t0.getTransactionContext().getRecord(tableName, rids.get(0)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5 * 2)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5 * 3)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() - 1)); + + assertEquals(prepare(t0.getTransNum(), + "acquire %s database IS", + "acquire %s database/tables.testTable1 IS", + "acquire %s database/tables.testTable1/30000000001 S", + "acquire %s database/tables.testTable1/30000000004 S", + "acquire %s database/tables.testTable1/30000000008 S", + "acquire %s database/tables.testTable1/30000000011 S", + "acquire-and-release %s database/tables.testTable1 S [database/tables.testTable1, " + + "database/tables.testTable1/30000000001, database/tables.testTable1/30000000004, " + + "database/tables.testTable1/30000000008, database/tables.testTable1/30000000011]" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testAutoEscalateX() { + String tableName = "testTable1"; + List rids = createTable(tableName, 18); + List values = TestUtils.createRecordWithAllTypes().getValues(); + + lockManager.startLog(); + + try(Transaction t0 = beginTransaction()) { + t0.getTransactionContext().updateRecord(tableName, values, rids.get(0)); + t0.getTransactionContext().deleteRecord(tableName, rids.get(rids.size() / 5)); + t0.getTransactionContext().updateRecord(tableName, values, rids.get(rids.size() / 5 * 2)); + t0.getTransactionContext().deleteRecord(tableName, rids.get(rids.size() / 5 * 3)); + t0.getTransactionContext().updateRecord(tableName, values, rids.get(rids.size() - 1)); + + assertEquals(prepare(t0.getTransNum(), + "acquire %s database IX", + "acquire %s database/tables.testTable1 IX", + "acquire %s database/tables.testTable1/30000000001 X", + "acquire %s database/tables.testTable1/30000000004 X", + "acquire %s database/tables.testTable1/30000000008 X", + "acquire %s database/tables.testTable1/30000000011 X", + "acquire-and-release %s database/tables.testTable1 X [database/tables.testTable1, " + + "database/tables.testTable1/30000000001, database/tables.testTable1/30000000004, " + + "database/tables.testTable1/30000000008, database/tables.testTable1/30000000011]" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testAutoEscalateSIX() { + String tableName = "testTable1"; + List rids = createTable(tableName, 18); + List values = TestUtils.createRecordWithAllTypes().getValues(); + + lockManager.startLog(); + + try(Transaction t0 = beginTransaction()) { + t0.getTransactionContext().getRecord(tableName, rids.get(0)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5 * 2)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5 * 3)); + t0.getTransactionContext().updateRecord(tableName, values, rids.get(rids.size() - 1)); + + assertEquals(prepare(t0.getTransNum(), + "acquire %s database IS", + "acquire %s database/tables.testTable1 IS", + "acquire %s database/tables.testTable1/30000000001 S", + "acquire %s database/tables.testTable1/30000000004 S", + "acquire %s database/tables.testTable1/30000000008 S", + "acquire %s database/tables.testTable1/30000000011 S", + "acquire-and-release %s database/tables.testTable1 S [database/tables.testTable1, " + + "database/tables.testTable1/30000000001, database/tables.testTable1/30000000004, " + + "database/tables.testTable1/30000000008, database/tables.testTable1/30000000011]", + "promote %s database IX", + "acquire-and-release %s database/tables.testTable1 SIX [database/tables.testTable1]", + "acquire %s database/tables.testTable1/30000000018 X" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testAutoEscalateDisabled() { + String tableName = "testTable1"; + List rids = createTable(tableName, 18); + + lockManager.startLog(); + + try(Transaction t0 = beginTransaction()) { + t0.getTransactionContext().getTable(tableName).disableAutoEscalate(); + + t0.getTransactionContext().getRecord(tableName, rids.get(0)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5 * 2)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() / 5 * 3)); + t0.getTransactionContext().getRecord(tableName, rids.get(rids.size() - 1)); + + assertEquals(prepare(t0.getTransNum(), + "acquire %s database IS", + "acquire %s database/tables.testTable1 IS", + "acquire %s database/tables.testTable1/30000000001 S", + "acquire %s database/tables.testTable1/30000000004 S", + "acquire %s database/tables.testTable1/30000000008 S", + "acquire %s database/tables.testTable1/30000000011 S", + "acquire %s database/tables.testTable1/30000000018 S" + ), removeMetadataLogs(lockManager.log)); + } + } + + @Test + @Category(PublicTests.class) + public void testLockTableMetadata() { + createTable("testTable1", 4); + + lockManager.startLog(); + + try(Transaction t = beginTransaction()) { + TransactionContext.setTransaction(t.getTransactionContext()); + + db.lockTableMetadata("tables.testTable1", LockType.S); + + assertEquals(prepare(t.getTransNum(), + "acquire %s database IS", + "acquire %s database/information_schema.tables IS", + "acquire %s database/information_schema.tables/10000000003 S" + ), lockManager.log); + + TransactionContext.unsetTransaction(); + } + + db.waitAllTransactions(); + lockManager.clearLog(); + lockManager.startLog(); + + try(Transaction t = beginTransaction()) { + TransactionContext.setTransaction(t.getTransactionContext()); + + db.lockTableMetadata("tables.testTable1", LockType.X); + + assertEquals(prepare(t.getTransNum(), + "acquire %s database IX", + "acquire %s database/information_schema.tables IX", + "acquire %s database/information_schema.tables/10000000003 X" + ), lockManager.log); + + TransactionContext.unsetTransaction(); + } + } + + @Test + @Category(PublicTests.class) + public void testLockIndexMetadata() { + createTableWithIndices("testTable1", 4, Collections.singletonList("int1")); + + lockManager.startLog(); + + try(Transaction t = beginTransaction()) { + TransactionContext.setTransaction(t.getTransactionContext()); + + db.lockIndexMetadata("testTable1,int1", LockType.S); + + assertEquals(prepare(t.getTransNum(), + "acquire %s database IS", + "acquire %s database/information_schema.indices IS", + "acquire %s database/information_schema.indices/20000000001 S" + ), lockManager.log); + + TransactionContext.unsetTransaction(); + } + + db.waitAllTransactions(); + lockManager.clearLog(); + lockManager.startLog(); + + try(Transaction t = beginTransaction()) { + TransactionContext.setTransaction(t.getTransactionContext()); + + db.lockIndexMetadata("testTable1,int1", LockType.X); + + assertEquals(prepare(t.getTransNum(), + "acquire %s database IX", + "acquire %s database/information_schema.indices IX", + "acquire %s database/information_schema.indices/20000000001 X" + ), lockManager.log); + + TransactionContext.unsetTransaction(); + } + } + + @Test + @Category(PublicTests.class) + public void testTableMetadataLockOnUse() { + lockManager.startLog(); + lockManager.suppressStatus(true); + + try(Transaction t = beginTransaction()) { + try { + t.getTransactionContext().getSchema("badTable"); + } catch (DatabaseException e) { /* do nothing */ } + + assertEquals(prepare(t.getTransNum(), + "acquire %s database IX", + "acquire %s database/information_schema.tables IX", + "acquire %s database/information_schema.tables/10000000003 X" + ), lockManager.log); + } + } + + @Test + @Category(PublicTests.class) + public void testCreateTableSimple() { + try(Transaction t = beginTransaction()) { + try { + t.getTransactionContext().getNumDataPages("testTable1"); + } catch (DatabaseException e) { /* do nothing */ } + } + db.waitAllTransactions(); + + lockManager.startLog(); + createTable("testTable1", 4); + + try(Transaction t = beginTransaction()) { + for (String x : lockManager.log) { + System.out.println(x); + } + assertSubsequence(prepare(t.getTransNum() - 1, + "acquire %s database IX", + "acquire %s database/information_schema.tables IX", + "acquire %s database/information_schema.tables/10000000003 X", + "acquire %s database/tables.testTable1 X" + ), lockManager.log); + } + } + + @Test + @Category(PublicTests.class) + public void testCreateIndexSimple() { + createTableWithIndices("testTable1", 4, Collections.emptyList()); + + try(Transaction t = beginTransaction()) { + try { + t.getTransactionContext().getTreeHeight("testTable1", "int1"); + } catch (DatabaseException e) { /* do nothing */ } + } + db.waitAllTransactions(); + + lockManager.startLog(); + + try(Transaction t = beginTransaction()) { + t.createIndex("testTable1", "int1", false); + assertSubsequence(prepare(t.getTransNum(), + "acquire %s database/information_schema.tables IS", + "acquire %s database/information_schema.tables/10000000003 S", + "acquire %s database/information_schema.indices IX", + "acquire %s database/information_schema.indices/20000000001 X", + "acquire %s database/indices.testTable1,int1 X" + ), lockManager.log); + } + } + + @Test + @Category(PublicTests.class) + public void testDropTableSimple() { + String tableName = "testTable1"; + createTable(tableName, 0); + lockManager.startLog(); + lockManager.suppressStatus(true); + + try(Transaction t0 = beginTransaction()) { + t0.dropTable(tableName); + + assertEquals(prepare(t0.getTransNum(), + "acquire %s database IX", + "acquire %s database/information_schema.tables IX", + "acquire %s database/information_schema.tables/10000000003 X", + "acquire %s database/tables.testTable1 X" + ), lockManager.log); + } + } + + @Test + @Category(PublicTests.class) + public void testDropIndexSimple() { + createTableWithIndices("testTable1", 4, Collections.singletonList("int1")); + lockManager.startLog(); + lockManager.suppressStatus(true); + + try(Transaction t0 = beginTransaction()) { + t0.dropIndex("testTable1", "int1"); + + assertSubsequence(prepare(t0.getTransNum(), + "acquire %s database IX", + "acquire %s database/information_schema.indices IX", + "acquire %s database/information_schema.indices/20000000001 X", + "acquire %s database/indices.testTable1,int1 X" + ), lockManager.log); + } + } + + @Test + @Category(PublicTests.class) + public void testDropAllTables() { + lockManager.startLog(); + + try(Transaction t0 = beginTransaction()) { + t0.dropAllTables(); + + assertEquals(prepare(t0.getTransNum(), + "acquire %s database X" + ), lockManager.log); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/TestDatabaseQueries.java b/src/test/java/edu/berkeley/cs186/database/TestDatabaseQueries.java new file mode 100644 index 0000000..35ed016 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/TestDatabaseQueries.java @@ -0,0 +1,164 @@ +package edu.berkeley.cs186.database; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.concurrency.DummyLockManager; +import edu.berkeley.cs186.database.databox.*; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.query.QueryPlan; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import static org.junit.Assert.*; + +@Category({HW99Tests.class}) +public class TestDatabaseQueries { + private Database database; + private Transaction transaction; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setup() throws IOException { + File tempDir = tempFolder.newFolder("myDb", "school"); + database = new Database(tempDir.getAbsolutePath(), 32, new DummyLockManager()); + database.setWorkMem(5); // B=5 + + createSchemas(); + readTuplesFromFiles(); + + transaction = database.beginTransaction(); + } + + @After + public void teardown() { + transaction.commit(); + database.close(); + } + + @Test + @Category(SystemTests.class) + public void testJoinStudentNamesWithClassNames() { + QueryPlan queryPlan = this.transaction.query("Students", "S"); + queryPlan.join("Enrollments", "E", "S.sid", "E.sid"); + queryPlan.join("Courses", "C", "E.cid", "C.cid"); + List columns = new ArrayList<>(); + columns.add("S.name"); + columns.add("C.name"); + queryPlan.project(columns); + + Iterator recordIterator = queryPlan.execute(); + + int count = 0; + while (recordIterator.hasNext()) { + recordIterator.next(); + count++; + } + + assertEquals(1000, count); + } + + private void createSchemas() { + List studentSchemaNames = new ArrayList<>(); + studentSchemaNames.add("sid"); + studentSchemaNames.add("name"); + studentSchemaNames.add("major"); + studentSchemaNames.add("gpa"); + + List studentSchemaTypes = new ArrayList<>(); + studentSchemaTypes.add(Type.intType()); + studentSchemaTypes.add(Type.stringType(20)); + studentSchemaTypes.add(Type.stringType(20)); + studentSchemaTypes.add(Type.floatType()); + + Schema studentSchema = new Schema(studentSchemaNames, studentSchemaTypes); + + try(Transaction t = database.beginTransaction()) { + t.createTable(studentSchema, "Students"); + + List courseSchemaNames = new ArrayList<>(); + courseSchemaNames.add("cid"); + courseSchemaNames.add("name"); + courseSchemaNames.add("department"); + + List courseSchemaTypes = new ArrayList<>(); + courseSchemaTypes.add(Type.intType()); + courseSchemaTypes.add(Type.stringType(20)); + courseSchemaTypes.add(Type.stringType(20)); + + Schema courseSchema = new Schema(courseSchemaNames, courseSchemaTypes); + + t.createTable(courseSchema, "Courses"); + + List enrollmentSchemaNames = new ArrayList<>(); + enrollmentSchemaNames.add("sid"); + enrollmentSchemaNames.add("cid"); + + List enrollmentSchemaTypes = new ArrayList<>(); + enrollmentSchemaTypes.add(Type.intType()); + enrollmentSchemaTypes.add(Type.intType()); + + Schema enrollmentSchema = new Schema(enrollmentSchemaNames, enrollmentSchemaTypes); + + t.createTable(enrollmentSchema, "Enrollments"); + } + } + + private void readTuplesFromFiles() throws IOException { + try(Transaction transaction = database.beginTransaction()) { + // read student tuples + List studentLines = Files.readAllLines(Paths.get("students.csv"), Charset.defaultCharset()); + + for (String line : studentLines) { + String[] splits = line.split(","); + List values = new ArrayList<>(); + + values.add(new IntDataBox(Integer.parseInt(splits[0]))); + values.add(new StringDataBox(splits[1].trim(), 20)); + values.add(new StringDataBox(splits[2].trim(), 20)); + values.add(new FloatDataBox(Float.parseFloat(splits[3]))); + + transaction.insert("Students", values); + } + + List courseLines = Files.readAllLines(Paths.get("courses.csv"), Charset.defaultCharset()); + + for (String line : courseLines) { + String[] splits = line.split(","); + List values = new ArrayList<>(); + + values.add(new IntDataBox(Integer.parseInt(splits[0]))); + values.add(new StringDataBox(splits[1].trim(), 20)); + values.add(new StringDataBox(splits[2].trim(), 20)); + + transaction.insert("Courses", values); + } + + List enrollmentLines = Files.readAllLines(Paths.get("enrollments.csv"), + Charset.defaultCharset()); + + for (String line : enrollmentLines) { + String[] splits = line.split(","); + List values = new ArrayList<>(); + + values.add(new IntDataBox(Integer.parseInt(splits[0]))); + values.add(new IntDataBox(Integer.parseInt(splits[1]))); + + transaction.insert("Enrollments", values); + } + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/TestUtils.java b/src/test/java/edu/berkeley/cs186/database/TestUtils.java new file mode 100644 index 0000000..0c0d8d3 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/TestUtils.java @@ -0,0 +1,117 @@ +package edu.berkeley.cs186.database; + +import edu.berkeley.cs186.database.databox.*; +import edu.berkeley.cs186.database.query.QueryPlanException; +import edu.berkeley.cs186.database.query.TestSourceOperator; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class TestUtils { + public static Schema createSchemaWithAllTypes() { + List names = Arrays.asList("bool", "int", "string", "float"); + List types = Arrays.asList(Type.boolType(), Type.intType(), + Type.stringType(1), Type.floatType()); + return new Schema(names, types); + } + + public static Schema createSchemaWithAllTypes(String prefix) { + List names = Arrays.asList(prefix + "bool", prefix + "int", prefix + "string", + prefix + "float"); + List types = Arrays.asList(Type.boolType(), Type.intType(), + Type.stringType(1), Type.floatType()); + return new Schema(names, types); + } + + public static Schema createSchemaWithTwoInts() { + List dataBoxes = new ArrayList(); + List fieldNames = new ArrayList(); + + dataBoxes.add(Type.intType()); + dataBoxes.add(Type.intType()); + + fieldNames.add("int1"); + fieldNames.add("int2"); + + return new Schema(fieldNames, dataBoxes); + } + + public static Schema createSchemaOfBool() { + List dataBoxes = new ArrayList(); + List fieldNames = new ArrayList(); + + dataBoxes.add(Type.boolType()); + + fieldNames.add("bool"); + + return new Schema(fieldNames, dataBoxes); + } + + public static Schema createSchemaOfString(int len) { + List dataBoxes = new ArrayList(); + List fieldNames = new ArrayList(); + + dataBoxes.add(Type.stringType(len)); + fieldNames.add("string"); + + return new Schema(fieldNames, dataBoxes); + } + + public static Record createRecordWithAllTypes() { + List dataValues = new ArrayList(); + dataValues.add(new BoolDataBox(true)); + dataValues.add(new IntDataBox(1)); + dataValues.add(new StringDataBox("a", 1)); + dataValues.add(new FloatDataBox((float) 1.2)); + + return new Record(dataValues); + } + + public static Record createRecordWithAllTypesWithValue(int val) { + List dataValues = new ArrayList(); + dataValues.add(new BoolDataBox(true)); + dataValues.add(new IntDataBox(val)); + dataValues.add(new StringDataBox("" + (char) (val % 79 + 0x30), 1)); + dataValues.add(new FloatDataBox((float) val)); + return new Record(dataValues); + } + + public static TestSourceOperator createTestSourceOperatorWithInts(List values) { + List columnNames = new ArrayList(); + columnNames.add("int"); + List columnTypes = new ArrayList(); + columnTypes.add(Type.intType()); + Schema schema = new Schema(columnNames, columnTypes); + + List recordList = new ArrayList(); + + for (int v : values) { + List recordValues = new ArrayList(); + recordValues.add(new IntDataBox(v)); + recordList.add(new Record(recordValues)); + } + + return new TestSourceOperator(recordList, schema); + } + + public static TestSourceOperator createTestSourceOperatorWithFloats(List values) { + List columnNames = new ArrayList(); + columnNames.add("float"); + List columnTypes = new ArrayList(); + columnTypes.add(Type.floatType()); + Schema schema = new Schema(columnNames, columnTypes); + + List recordList = new ArrayList(); + + for (float v : values) { + List recordValues = new ArrayList(); + recordValues.add(new FloatDataBox(v)); + recordList.add(new Record(recordValues)); + } + + return new TestSourceOperator(recordList, schema); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/TimeoutScaling.java b/src/test/java/edu/berkeley/cs186/database/TimeoutScaling.java new file mode 100644 index 0000000..c2f22cc --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/TimeoutScaling.java @@ -0,0 +1,6 @@ +package edu.berkeley.cs186.database; + +public final class TimeoutScaling { + // How much to scale test timeouts by. + public static final double factor = 1.0; +} diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW0Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW0Tests.java new file mode 100644 index 0000000..97c6b73 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW0Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW0Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW2Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW2Tests.java new file mode 100644 index 0000000..ddc6056 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW2Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW2Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW3Part1Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW3Part1Tests.java new file mode 100644 index 0000000..8934aa0 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW3Part1Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW3Part1Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW3Part2Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW3Part2Tests.java new file mode 100644 index 0000000..8cf1b2f --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW3Part2Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW3Part2Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW3Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW3Tests.java new file mode 100644 index 0000000..5e56534 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW3Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW3Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW4Part1Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW4Part1Tests.java new file mode 100644 index 0000000..6b6b788 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW4Part1Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW4Part1Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW4Part2Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW4Part2Tests.java new file mode 100644 index 0000000..4d9bc14 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW4Part2Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW4Part2Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW4Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW4Tests.java new file mode 100644 index 0000000..b3ec90e --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW4Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW4Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW5Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW5Tests.java new file mode 100644 index 0000000..c013aef --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW5Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW5Tests extends HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HW99Tests.java b/src/test/java/edu/berkeley/cs186/database/categories/HW99Tests.java new file mode 100644 index 0000000..631029c --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HW99Tests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HW99Tests extends HWTests { /* category marker for non-homework tests */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HWTests.java b/src/test/java/edu/berkeley/cs186/database/categories/HWTests.java new file mode 100644 index 0000000..822351b --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HWTests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HWTests { /* category marker */ } \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/categories/HiddenTests.java b/src/test/java/edu/berkeley/cs186/database/categories/HiddenTests.java new file mode 100644 index 0000000..3444be8 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/HiddenTests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface HiddenTests { /* category marker */ } diff --git a/src/test/java/edu/berkeley/cs186/database/categories/PublicTests.java b/src/test/java/edu/berkeley/cs186/database/categories/PublicTests.java new file mode 100644 index 0000000..51d56a9 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/PublicTests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface PublicTests { /* category marker */ } diff --git a/src/test/java/edu/berkeley/cs186/database/categories/StudentTestRunner.java b/src/test/java/edu/berkeley/cs186/database/categories/StudentTestRunner.java new file mode 100644 index 0000000..155235b --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/StudentTestRunner.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface StudentTestRunner { /* category marker */ } diff --git a/src/test/java/edu/berkeley/cs186/database/categories/StudentTests.java b/src/test/java/edu/berkeley/cs186/database/categories/StudentTests.java new file mode 100644 index 0000000..ed72f4e --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/StudentTests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface StudentTests { /* category marker */ } diff --git a/src/test/java/edu/berkeley/cs186/database/categories/SystemTests.java b/src/test/java/edu/berkeley/cs186/database/categories/SystemTests.java new file mode 100644 index 0000000..fcb0e2c --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/categories/SystemTests.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.categories; + +public interface SystemTests { /* category marker */ } diff --git a/src/test/java/edu/berkeley/cs186/database/common/TestBits.java b/src/test/java/edu/berkeley/cs186/database/common/TestBits.java new file mode 100644 index 0000000..d9027ee --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/common/TestBits.java @@ -0,0 +1,121 @@ +package edu.berkeley.cs186.database.common; + +import edu.berkeley.cs186.database.categories.*; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestBits { + @Test + public void testGetBitOnByte() { + byte b = 0b01101011; + assertEquals(Bits.Bit.ZERO, Bits.getBit(b, 0)); + assertEquals(Bits.Bit.ONE, Bits.getBit(b, 1)); + assertEquals(Bits.Bit.ONE, Bits.getBit(b, 2)); + assertEquals(Bits.Bit.ZERO, Bits.getBit(b, 3)); + assertEquals(Bits.Bit.ONE, Bits.getBit(b, 4)); + assertEquals(Bits.Bit.ZERO, Bits.getBit(b, 5)); + assertEquals(Bits.Bit.ONE, Bits.getBit(b, 6)); + assertEquals(Bits.Bit.ONE, Bits.getBit(b, 7)); + } + + @Test + public void testGetBitOnBytes() { + byte[] bytes = {0b01101011, 0b01001101}; + + assertEquals(Bits.Bit.ZERO, Bits.getBit(bytes, 0)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 1)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 2)); + assertEquals(Bits.Bit.ZERO, Bits.getBit(bytes, 3)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 4)); + assertEquals(Bits.Bit.ZERO, Bits.getBit(bytes, 5)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 6)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 7)); + + assertEquals(Bits.Bit.ZERO, Bits.getBit(bytes, 8)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 9)); + assertEquals(Bits.Bit.ZERO, Bits.getBit(bytes, 10)); + assertEquals(Bits.Bit.ZERO, Bits.getBit(bytes, 11)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 12)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 13)); + assertEquals(Bits.Bit.ZERO, Bits.getBit(bytes, 14)); + assertEquals(Bits.Bit.ONE, Bits.getBit(bytes, 15)); + } + + @Test + public void testSetBitOnByte() { + assertEquals((byte) 0b10000000, Bits.setBit((byte) 0b00000000, 0, Bits.Bit.ONE)); + assertEquals((byte) 0b01000000, Bits.setBit((byte) 0b00000000, 1, Bits.Bit.ONE)); + assertEquals((byte) 0b00100000, Bits.setBit((byte) 0b00000000, 2, Bits.Bit.ONE)); + assertEquals((byte) 0b00010000, Bits.setBit((byte) 0b00000000, 3, Bits.Bit.ONE)); + assertEquals((byte) 0b00001000, Bits.setBit((byte) 0b00000000, 4, Bits.Bit.ONE)); + assertEquals((byte) 0b00000100, Bits.setBit((byte) 0b00000000, 5, Bits.Bit.ONE)); + assertEquals((byte) 0b00000010, Bits.setBit((byte) 0b00000000, 6, Bits.Bit.ONE)); + assertEquals((byte) 0b00000001, Bits.setBit((byte) 0b00000000, 7, Bits.Bit.ONE)); + + assertEquals((byte) 0b01111111, Bits.setBit((byte) 0b11111111, 0, Bits.Bit.ZERO)); + assertEquals((byte) 0b10111111, Bits.setBit((byte) 0b11111111, 1, Bits.Bit.ZERO)); + assertEquals((byte) 0b11011111, Bits.setBit((byte) 0b11111111, 2, Bits.Bit.ZERO)); + assertEquals((byte) 0b11101111, Bits.setBit((byte) 0b11111111, 3, Bits.Bit.ZERO)); + assertEquals((byte) 0b11110111, Bits.setBit((byte) 0b11111111, 4, Bits.Bit.ZERO)); + assertEquals((byte) 0b11111011, Bits.setBit((byte) 0b11111111, 5, Bits.Bit.ZERO)); + assertEquals((byte) 0b11111101, Bits.setBit((byte) 0b11111111, 6, Bits.Bit.ZERO)); + assertEquals((byte) 0b11111110, Bits.setBit((byte) 0b11111111, 7, Bits.Bit.ZERO)); + } + + @Test + public void testSetBitOnBytes() { + byte[] bytes = {0b00000000, 0b00000000}; + + // Write 1's. + byte[][] expectedsOne = { + {(byte) 0b10000000, (byte) 0b00000000}, + {(byte) 0b11000000, (byte) 0b00000000}, + {(byte) 0b11100000, (byte) 0b00000000}, + {(byte) 0b11110000, (byte) 0b00000000}, + {(byte) 0b11111000, (byte) 0b00000000}, + {(byte) 0b11111100, (byte) 0b00000000}, + {(byte) 0b11111110, (byte) 0b00000000}, + {(byte) 0b11111111, (byte) 0b00000000}, + {(byte) 0b11111111, (byte) 0b10000000}, + {(byte) 0b11111111, (byte) 0b11000000}, + {(byte) 0b11111111, (byte) 0b11100000}, + {(byte) 0b11111111, (byte) 0b11110000}, + {(byte) 0b11111111, (byte) 0b11111000}, + {(byte) 0b11111111, (byte) 0b11111100}, + {(byte) 0b11111111, (byte) 0b11111110}, + {(byte) 0b11111111, (byte) 0b11111111}, + }; + for (int i = 0; i < 16; ++i) { + Bits.setBit(bytes, i, Bits.Bit.ONE); + assertArrayEquals(expectedsOne[i], bytes); + } + + // Write 0's. + byte[][] expectedsZero = { + {(byte) 0b01111111, (byte) 0b11111111}, + {(byte) 0b00111111, (byte) 0b11111111}, + {(byte) 0b00011111, (byte) 0b11111111}, + {(byte) 0b00001111, (byte) 0b11111111}, + {(byte) 0b00000111, (byte) 0b11111111}, + {(byte) 0b00000011, (byte) 0b11111111}, + {(byte) 0b00000001, (byte) 0b11111111}, + {(byte) 0b00000000, (byte) 0b11111111}, + {(byte) 0b00000000, (byte) 0b01111111}, + {(byte) 0b00000000, (byte) 0b00111111}, + {(byte) 0b00000000, (byte) 0b00011111}, + {(byte) 0b00000000, (byte) 0b00001111}, + {(byte) 0b00000000, (byte) 0b00000111}, + {(byte) 0b00000000, (byte) 0b00000011}, + {(byte) 0b00000000, (byte) 0b00000001}, + {(byte) 0b00000000, (byte) 0b00000000}, + }; + for (int i = 0; i < 16; ++i) { + Bits.setBit(bytes, i, Bits.Bit.ZERO); + assertArrayEquals(expectedsZero[i], bytes); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/concurrency/DeterministicRunner.java b/src/test/java/edu/berkeley/cs186/database/concurrency/DeterministicRunner.java new file mode 100644 index 0000000..acfc430 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/concurrency/DeterministicRunner.java @@ -0,0 +1,134 @@ +package edu.berkeley.cs186.database.concurrency; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** + * A utility class for running code over multiple threads, in a specific order. Any + * exceptions/errors thrown in a child thread are rethrown in the main thread. + */ +public class DeterministicRunner { + private final Worker[] workers; + private Throwable error = null; + + private class Worker implements Runnable { + private Thread thread; + private final ReentrantLock lock = new ReentrantLock(); + private final Condition sleepCondition = lock.newCondition(); + private final Condition wakeCondition = lock.newCondition(); + private final AtomicBoolean awake = new AtomicBoolean(false); + private final AtomicBoolean ready = new AtomicBoolean(false); + private Runnable nextTask = null; + + public Worker() { + this.thread = new Thread(this); + } + + @Override + public void run() { + try { + sleep(); + while (nextTask != null) { + nextTask.run(); + sleep(); + } + } catch (Throwable throwable) { + error = throwable; + } + } + + private void sleep() { + lock.lock(); + try { + while (!ready.get()) { + sleepCondition.awaitUninterruptibly(); + } + } finally { + awake.set(true); + ready.set(false); + wakeCondition.signal(); + lock.unlock(); + } + } + + public void start() { + thread.start(); + } + + public void runTask(Runnable next) { + lock.lock(); + try { + nextTask = next; + ready.set(true); + sleepCondition.signal(); + } finally { + lock.unlock(); + } + lock.lock(); + try { + while (!awake.get()) { + wakeCondition.awaitUninterruptibly(); + } + awake.set(false); + } finally { + lock.unlock(); + } + while (thread.getState() != Thread.State.WAITING && thread.getState() != Thread.State.TERMINATED && + error == null) { + // return when either we finished the task (and went back to sleep) + // or when the task caused the thread to block + Thread.yield(); + } + } + + public void join() throws InterruptedException { + lock.lock(); + try { + nextTask = null; + ready.set(true); + sleepCondition.signal(); + } finally { + lock.unlock(); + } + thread.join(); + } + } + + public DeterministicRunner(int numWorkers) { + this.workers = new Worker[numWorkers]; + for (int i = 0; i < numWorkers; ++i) { + this.workers[i] = new Worker(); + this.workers[i].start(); + } + } + + public void run(int thread, Runnable task) { + error = null; + this.workers[thread].runTask(task); + if (error != null) { + rethrow(error); + } + } + + public void join(int thread) { + try { + this.workers[thread].join(); + } catch (Throwable t) { + rethrow(t); + } + } + + public void joinAll() { + for (int i = 0; i < this.workers.length; ++i) { + join(i); + } + } + + @SuppressWarnings("unchecked") + private static void rethrow(Throwable t) throws T { + // rethrows checked exceptions as unchecked + throw (T) t; + } + +} diff --git a/src/test/java/edu/berkeley/cs186/database/concurrency/DummyTransactionContext.java b/src/test/java/edu/berkeley/cs186/database/concurrency/DummyTransactionContext.java new file mode 100644 index 0000000..c557f42 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/concurrency/DummyTransactionContext.java @@ -0,0 +1,238 @@ +package edu.berkeley.cs186.database.concurrency; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.UnaryOperator; + +import edu.berkeley.cs186.database.AbstractTransactionContext; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.index.BPlusTreeMetadata; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.query.QueryPlan; +import edu.berkeley.cs186.database.table.*; +import edu.berkeley.cs186.database.table.stats.TableStats; + +/** + * A dummy transaction class that only supports checking/setting active/blocked + * status. Used for testing locking code without requiring an instance + * of the database. + */ +public class DummyTransactionContext extends AbstractTransactionContext { + private long tNum; + private LoggingLockManager lockManager; + private boolean active = true; + + public DummyTransactionContext(LoggingLockManager lockManager, long tNum) { + this.lockManager = lockManager; + this.tNum = tNum; + } + + @Override + public long getTransNum() { + return this.tNum; + } + + @Override + public String createTempTable(Schema schema) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public void deleteAllTempTables() { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public void setAliasMap(Map aliasMap) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public void clearAliasMap() { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public boolean indexExists(String tableName, String columnName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public Iterator sortedScan(String tableName, String columnName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public Iterator sortedScanFrom(String tableName, String columnName, + DataBox startValue) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public Iterator lookupKey(String tableName, String columnName, + DataBox key) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public boolean contains(String tableName, String columnName, DataBox key) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public RecordId addRecord(String tableName, List values) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public int getWorkMemSize() { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public RecordId deleteRecord(String tableName, RecordId rid) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public Record getRecord(String tableName, RecordId rid) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public RecordIterator getRecordIterator(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public RecordId updateRecord(String tableName, List values, + RecordId rid) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public BacktrackingIterator getPageIterator(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + public BacktrackingIterator getBlockIterator(String tableName, + Page[] block) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + public BacktrackingIterator getBlockIterator(String tableName, + BacktrackingIterator block) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public BacktrackingIterator getBlockIterator(String tableName, Iterator block, + int maxPages) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public void runUpdateRecordWhere(String tableName, String targetColumnName, + UnaryOperator targetValue, + String predColumnName, PredicateOperator predOperator, DataBox predValue) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public void runDeleteRecordWhere(String tableName, String predColumnName, + PredicateOperator predOperator, DataBox predValue) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public TableStats getStats(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public int getNumDataPages(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public int getNumEntriesPerPage(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + public byte[] readPageHeader(String tableName, Page p) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + public int getPageHeaderSize(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public int getEntrySize(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public long getNumRecords(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public int getTreeOrder(String tableName, String columnName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public int getTreeHeight(String tableName, String columnName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public Schema getSchema(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public Schema getFullyQualifiedSchema(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public Table getTable(String tableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + public void deleteTempTable(String tempTableName) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public void updateIndexMetadata(BPlusTreeMetadata metadata) { + throw new UnsupportedOperationException("dummy transaction cannot do this"); + } + + @Override + public void block() { + lockManager.emit("block " + tNum); + super.block(); + } + + @Override + public void unblock() { + lockManager.emit("unblock " + tNum); + super.unblock(); + Thread.yield(); + } + + @Override + public void close() {} + + @Override + public String toString() { + return "Dummy Transaction #" + tNum; + } +} + diff --git a/src/test/java/edu/berkeley/cs186/database/concurrency/LoggingLockContext.java b/src/test/java/edu/berkeley/cs186/database/concurrency/LoggingLockContext.java new file mode 100644 index 0000000..608094c --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/concurrency/LoggingLockContext.java @@ -0,0 +1,63 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.common.Pair; + +public class LoggingLockContext extends LockContext { + private boolean allowDisable = true; + + LoggingLockContext(LoggingLockManager lockman, LockContext parent, Pair name) { + super(lockman, parent, name); + } + + private LoggingLockContext(LoggingLockManager lockman, LockContext parent, Pair name, + boolean readonly) { + super(lockman, parent, name, readonly); + } + + /** + * Disables locking children. This causes all child contexts of this context + * to be readonly. This is used for indices and temporary tables (where + * we disallow finer-grain locks), the former due to complexity locking + * B+ trees, and the latter due to the fact that temporary tables are only + * accessible to one transaction, so finer-grain locks make no sense. + */ + @Override + public synchronized void disableChildLocks() { + if (this.allowDisable) { + super.disableChildLocks(); + } + ((LoggingLockManager) lockman).emit("disable-children " + name); + } + + /** + * Gets the context for the child with name NAME (with a readable version READABLE). + */ + @Override + public synchronized LockContext childContext(String readable, long name) { + LockContext temp = new LoggingLockContext((LoggingLockManager) lockman, this, new Pair<>(readable, + name), + this.childLocksDisabled || this.readonly); + LockContext child = this.children.putIfAbsent(name, temp); + if (child == null) { + child = temp; + } + return child; + } + + /** + * Sets the capacity (number of children). + */ + @Override + public synchronized void capacity(int capacity) { + int oldCapacity = super.capacity; + super.capacity(capacity); + if (oldCapacity != capacity) { + ((LoggingLockManager) lockman).emit("set-capacity " + name + " " + capacity); + } + } + + public synchronized void allowDisableChildLocks(boolean allow) { + this.allowDisable = allow; + } +} + diff --git a/src/test/java/edu/berkeley/cs186/database/concurrency/LoggingLockManager.java b/src/test/java/edu/berkeley/cs186/database/concurrency/LoggingLockManager.java new file mode 100644 index 0000000..e7e8941 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/concurrency/LoggingLockManager.java @@ -0,0 +1,137 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.Pair; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +public class LoggingLockManager extends LockManager { + public List log = Collections.synchronizedList(new ArrayList<>()); + private boolean logging = false; + private boolean suppressInternal = true; + private boolean suppressStatus = false; + private Map contexts = new HashMap<>(); + private Map loggingOverride = new ConcurrentHashMap<>(); + + @Override + public synchronized LockContext context(String readable, long name) { + if (!contexts.containsKey(name)) { + contexts.put(name, new LoggingLockContext(this, null, new Pair<>(readable, name))); + } + return contexts.get(name); + } + + @Override + public void acquireAndRelease(TransactionContext transaction, ResourceName name, + LockType lockType, List releaseLocks) { + StringBuilder estr = new StringBuilder("acquire-and-release "); + estr.append(transaction.getTransNum()).append(' ').append(name).append(' ').append(lockType); + releaseLocks.sort(Comparator.comparing(ResourceName::toString)); + estr.append(" ["); + boolean first = true; + for (ResourceName n : releaseLocks) { + if (!first) { + estr.append(", "); + } + estr.append(n); + first = false; + } + estr.append(']'); + emit(estr.toString()); + + Boolean[] oldOverride = new Boolean[1]; + loggingOverride.compute(Thread.currentThread().getId(), (id, old) -> { + oldOverride[0] = old; + return !suppressInternal; + }); + try { + super.acquireAndRelease(transaction, name, lockType, releaseLocks); + } finally { + loggingOverride.compute(Thread.currentThread().getId(), (id, old) -> oldOverride[0]); + } + } + + @Override + public void acquire(TransactionContext transaction, ResourceName name, LockType type) { + emit("acquire " + transaction.getTransNum() + " " + name + " " + type); + + Boolean[] oldOverride = new Boolean[1]; + loggingOverride.compute(Thread.currentThread().getId(), (id, old) -> { + oldOverride[0] = old; + return !suppressInternal; + }); + try { + super.acquire(transaction, name, type); + } finally { + loggingOverride.compute(Thread.currentThread().getId(), (id, old) -> oldOverride[0]); + } + } + + @Override + public void release(TransactionContext transaction, ResourceName name) { + emit("release " + transaction.getTransNum() + " " + name); + + Boolean[] oldOverride = new Boolean[1]; + loggingOverride.compute(Thread.currentThread().getId(), (id, old) -> { + oldOverride[0] = old; + return !suppressInternal; + }); + try { + super.release(transaction, name); + } finally { + loggingOverride.compute(Thread.currentThread().getId(), (id, old) -> oldOverride[0]); + } + } + + @Override + public void promote(TransactionContext transaction, ResourceName name, LockType newLockType) { + emit("promote " + transaction.getTransNum() + " " + name + " " + newLockType); + + Boolean[] oldOverride = new Boolean[1]; + loggingOverride.compute(Thread.currentThread().getId(), (id, old) -> { + oldOverride[0] = old; + return !suppressInternal; + }); + try { + super.promote(transaction, name, newLockType); + } finally { + loggingOverride.compute(Thread.currentThread().getId(), (id, old) -> oldOverride[0]); + } + } + + public void startLog() { + logging = true; + } + + public void endLog() { + logging = false; + } + + public void clearLog() { + log.clear(); + } + + public boolean isLogging() { + return logging; + } + + void suppressInternals(boolean toggle) { + suppressInternal = toggle; + } + + public void suppressStatus(boolean toggle) { + suppressStatus = toggle; + } + + void emit(String s) { + long tid = Thread.currentThread().getId(); + if (suppressStatus && !s.startsWith("acquire") && !s.startsWith("promote") && + !s.startsWith("release")) { + return; + } + if ((loggingOverride.containsKey(tid) ? loggingOverride.get(tid) : logging)) { + log.add(s); + } + } +} \ No newline at end of file diff --git a/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockContext.java b/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockContext.java new file mode 100644 index 0000000..5a42e3a --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockContext.java @@ -0,0 +1,399 @@ +package edu.berkeley.cs186.database.concurrency; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import edu.berkeley.cs186.database.AbstractTransactionContext; +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.Pair; + +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import static org.junit.Assert.*; + +@Category({HW4Tests.class, HW4Part1Tests.class}) +public class TestLockContext { + private LoggingLockManager lockManager; + + private LockContext dbLockContext; + private LockContext tableLockContext; + private LockContext pageLockContext; + + private TransactionContext[] transactions; + + // 1 second per test + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + @Before + public void setUp() { + lockManager = new LoggingLockManager(); + + dbLockContext = lockManager.databaseContext(); + tableLockContext = dbLockContext.childContext("table1", 1); + pageLockContext = tableLockContext.childContext("page1", 1); + + transactions = new TransactionContext[8]; + for (int i = 0; i < transactions.length; i++) { + transactions[i] = new DummyTransactionContext(lockManager, i); + } + } + + @Test + @Category(PublicTests.class) + public void testSimpleAcquireFail() { + dbLockContext.acquire(transactions[0], LockType.IS); + try { + tableLockContext.acquire(transactions[0], LockType.X); + fail(); + } catch (InvalidLockException e) { + // do nothing + } + } + + @Test + @Category(PublicTests.class) + public void testSimpleAcquirePass() { + dbLockContext.acquire(transactions[0], LockType.IS); + tableLockContext.acquire(transactions[0], LockType.S); + Assert.assertEquals(Arrays.asList(new Lock(dbLockContext.getResourceName(), LockType.IS, 0L), + new Lock(tableLockContext.getResourceName(), LockType.S, 0L)), + lockManager.getLocks(transactions[0])); + } + + @Test + @Category(PublicTests.class) + public void testTreeAcquirePass() { + dbLockContext.acquire(transactions[0], LockType.IX); + tableLockContext.acquire(transactions[0], LockType.IS); + pageLockContext.acquire(transactions[0], LockType.S); + + Assert.assertEquals(Arrays.asList(new Lock(dbLockContext.getResourceName(), LockType.IX, 0L), + new Lock(tableLockContext.getResourceName(), LockType.IS, 0L), + new Lock(pageLockContext.getResourceName(), LockType.S, 0L)), + lockManager.getLocks(transactions[0])); + } + + @Test + @Category(PublicTests.class) + public void testSimpleReleasePass() { + dbLockContext.acquire(transactions[0], LockType.IS); + tableLockContext.acquire(transactions[0], LockType.S); + tableLockContext.release(transactions[0]); + + Assert.assertEquals(Collections.singletonList(new Lock(dbLockContext.getResourceName(), LockType.IS, + 0L)), + lockManager.getLocks(transactions[0])); + } + + @Test + @Category(PublicTests.class) + public void testSimpleReleaseFail() { + dbLockContext.acquire(transactions[0], LockType.IS); + tableLockContext.acquire(transactions[0], LockType.S); + try { + dbLockContext.release(transactions[0]); + fail(); + } catch (InvalidLockException e) { + // do nothing + } + } + + @Test + @Category(PublicTests.class) + public void testSharedPage() { + DeterministicRunner runner = new DeterministicRunner(2); + + TransactionContext t1 = transactions[1]; + TransactionContext t2 = transactions[2]; + + LockContext r0 = tableLockContext; + LockContext r1 = pageLockContext; + + runner.run(0, () -> dbLockContext.acquire(t1, LockType.IS)); + runner.run(1, () -> dbLockContext.acquire(t2, LockType.IS)); + runner.run(0, () -> r0.acquire(t1, LockType.IS)); + runner.run(1, () -> r0.acquire(t2, LockType.IS)); + runner.run(0, () -> r1.acquire(t1, LockType.S)); + runner.run(1, () -> r1.acquire(t2, LockType.S)); + + assertTrue(TestLockManager.holds(lockManager, t1, r0.getResourceName(), LockType.IS)); + assertTrue(TestLockManager.holds(lockManager, t2, r0.getResourceName(), LockType.IS)); + assertTrue(TestLockManager.holds(lockManager, t1, r1.getResourceName(), LockType.S)); + assertTrue(TestLockManager.holds(lockManager, t2, r1.getResourceName(), LockType.S)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSandIS() { + DeterministicRunner runner = new DeterministicRunner(2); + + TransactionContext t1 = transactions[1]; + TransactionContext t2 = transactions[2]; + + LockContext r0 = dbLockContext; + LockContext r1 = tableLockContext; + + runner.run(0, () -> r0.acquire(t1, LockType.S)); + runner.run(1, () -> r0.acquire(t2, LockType.IS)); + runner.run(1, () -> r1.acquire(t2, LockType.S)); + runner.run(0, () -> r0.release(t1)); + + assertTrue(TestLockManager.holds(lockManager, t2, r0.getResourceName(), LockType.IS)); + assertTrue(TestLockManager.holds(lockManager, t2, r1.getResourceName(), LockType.S)); + assertFalse(TestLockManager.holds(lockManager, t1, r0.getResourceName(), LockType.S)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSharedIntentConflict() { + DeterministicRunner runner = new DeterministicRunner(2); + + TransactionContext t1 = transactions[1]; + TransactionContext t2 = transactions[2]; + + LockContext r0 = dbLockContext; + LockContext r1 = tableLockContext; + + runner.run(0, () -> r0.acquire(t1, LockType.IS)); + runner.run(1, () -> r0.acquire(t2, LockType.IX)); + runner.run(0, () -> r1.acquire(t1, LockType.S)); + runner.run(1, () -> r1.acquire(t2, LockType.X)); + + assertTrue(TestLockManager.holds(lockManager, t1, r0.getResourceName(), LockType.IS)); + assertTrue(TestLockManager.holds(lockManager, t2, r0.getResourceName(), LockType.IX)); + assertTrue(TestLockManager.holds(lockManager, t1, r1.getResourceName(), LockType.S)); + assertFalse(TestLockManager.holds(lockManager, t2, r1.getResourceName(), LockType.X)); + + runner.join(0); + } + + @Test + @Category(PublicTests.class) + public void testSharedIntentConflictRelease() { + DeterministicRunner runner = new DeterministicRunner(2); + + TransactionContext t1 = transactions[1]; + TransactionContext t2 = transactions[2]; + + LockContext r0 = dbLockContext; + LockContext r1 = tableLockContext; + + runner.run(0, () -> r0.acquire(t1, LockType.IS)); + runner.run(1, () -> r0.acquire(t2, LockType.IX)); + runner.run(0, () -> r1.acquire(t1, LockType.S)); + runner.run(1, () -> r1.acquire(t2, LockType.X)); + runner.run(0, () -> r1.release(t1)); + + assertTrue(TestLockManager.holds(lockManager, t1, r0.getResourceName(), LockType.IS)); + assertTrue(TestLockManager.holds(lockManager, t2, r0.getResourceName(), LockType.IX)); + assertFalse(TestLockManager.holds(lockManager, t1, r1.getResourceName(), LockType.S)); + assertTrue(TestLockManager.holds(lockManager, t2, r1.getResourceName(), LockType.X)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSimplePromote() { + TransactionContext t1 = transactions[1]; + dbLockContext.acquire(t1, LockType.S); + dbLockContext.promote(t1, LockType.X); + assertTrue(TestLockManager.holds(lockManager, t1, dbLockContext.getResourceName(), LockType.X)); + } + + @Test + @Category(PublicTests.class) + public void testEscalateFail() { + TransactionContext t1 = transactions[1]; + + LockContext r0 = dbLockContext; + + try { + r0.escalate(t1); + fail(); + } catch (NoLockHeldException e) { + // do nothing + } + } + + @Test + @Category(PublicTests.class) + public void testEscalateISS() { + TransactionContext t1 = transactions[1]; + + LockContext r0 = dbLockContext; + + r0.acquire(t1, LockType.IS); + r0.escalate(t1); + assertTrue(TestLockManager.holds(lockManager, t1, r0.getResourceName(), LockType.S)); + } + + @Test + @Category(PublicTests.class) + public void testEscalateIXX() { + TransactionContext t1 = transactions[1]; + + LockContext r0 = dbLockContext; + + r0.acquire(t1, LockType.IX); + r0.escalate(t1); + assertTrue(TestLockManager.holds(lockManager, t1, r0.getResourceName(), LockType.X)); + } + + @Test + @Category(PublicTests.class) + public void testEscalateIdempotent() { + TransactionContext t1 = transactions[1]; + + LockContext r0 = dbLockContext; + + r0.acquire(t1, LockType.IS); + r0.escalate(t1); + lockManager.startLog(); + r0.escalate(t1); + r0.escalate(t1); + r0.escalate(t1); + assertEquals(Collections.emptyList(), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testEscalateS() { + TransactionContext t1 = transactions[1]; + + LockContext r0 = dbLockContext; + LockContext r1 = tableLockContext; + + r0.acquire(t1, LockType.IS); + r1.acquire(t1, LockType.S); + r0.escalate(t1); + + assertTrue(TestLockManager.holds(lockManager, t1, r0.getResourceName(), LockType.S)); + assertFalse(TestLockManager.holds(lockManager, t1, r1.getResourceName(), LockType.S)); + } + + @Test + @Category(PublicTests.class) + public void testEscalateMultipleS() { + TransactionContext t1 = transactions[1]; + + LockContext r0 = dbLockContext; + LockContext r1 = tableLockContext; + LockContext r2 = dbLockContext.childContext("table2", 2); + LockContext r3 = dbLockContext.childContext("table3", 3); + + r0.capacity(4); + + r0.acquire(t1, LockType.IS); + r1.acquire(t1, LockType.S); + r2.acquire(t1, LockType.IS); + r3.acquire(t1, LockType.S); + + assertEquals(3.0 / 4, r0.saturation(t1), 1e-6); + r0.escalate(t1); + assertEquals(0.0, r0.saturation(t1), 1e-6); + + assertTrue(TestLockManager.holds(lockManager, t1, r0.getResourceName(), LockType.S)); + assertFalse(TestLockManager.holds(lockManager, t1, r1.getResourceName(), LockType.S)); + assertFalse(TestLockManager.holds(lockManager, t1, r2.getResourceName(), LockType.IS)); + assertFalse(TestLockManager.holds(lockManager, t1, r3.getResourceName(), LockType.S)); + } + + @Test + @Category(PublicTests.class) + public void testGetLockType() { + DeterministicRunner runner = new DeterministicRunner(4); + + TransactionContext t1 = transactions[1]; + TransactionContext t2 = transactions[2]; + TransactionContext t3 = transactions[3]; + TransactionContext t4 = transactions[4]; + + runner.run(0, () -> dbLockContext.acquire(t1, LockType.S)); + runner.run(1, () -> dbLockContext.acquire(t2, LockType.IS)); + runner.run(2, () -> dbLockContext.acquire(t3, LockType.IS)); + runner.run(3, () -> dbLockContext.acquire(t4, LockType.IS)); + + runner.run(1, () -> tableLockContext.acquire(t2, LockType.S)); + runner.run(2, () -> tableLockContext.acquire(t3, LockType.IS)); + + runner.run(2, () -> pageLockContext.acquire(t3, LockType.S)); + + assertEquals(LockType.S, pageLockContext.getEffectiveLockType(t1)); + assertEquals(LockType.S, pageLockContext.getEffectiveLockType(t2)); + assertEquals(LockType.S, pageLockContext.getEffectiveLockType(t3)); + assertEquals(LockType.NL, pageLockContext.getEffectiveLockType(t4)); + assertEquals(LockType.NL, pageLockContext.getExplicitLockType(t1)); + assertEquals(LockType.NL, pageLockContext.getExplicitLockType(t2)); + assertEquals(LockType.S, pageLockContext.getExplicitLockType(t3)); + assertEquals(LockType.NL, pageLockContext.getExplicitLockType(t4)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testReadonly() { + dbLockContext.disableChildLocks(); + LockContext tableContext = dbLockContext.childContext("table2", 2); + TransactionContext t1 = transactions[1]; + dbLockContext.acquire(t1, LockType.IX); + try { + tableContext.acquire(t1, LockType.IX); + fail(); + } catch (UnsupportedOperationException e) { + // do nothing + } + try { + tableContext.release(t1); + fail(); + } catch (UnsupportedOperationException e) { + // do nothing + } + try { + tableContext.promote(t1, LockType.IX); + fail(); + } catch (UnsupportedOperationException e) { + // do nothing + } + try { + tableContext.escalate(t1); + fail(); + } catch (UnsupportedOperationException e) { + // do nothing + } + } + + @Test + @Category(PublicTests.class) + public void testSaturation() { + LockContext tableContext = dbLockContext.childContext("table2", 2); + TransactionContext t1 = transactions[1]; + dbLockContext.capacity(10); + dbLockContext.acquire(t1, LockType.IX); + tableContext.acquire(t1, LockType.IS); + assertEquals(0.1, dbLockContext.saturation(t1), 1E-6); + tableContext.promote(t1, LockType.IX); + assertEquals(0.1, dbLockContext.saturation(t1), 1E-6); + tableContext.release(t1); + assertEquals(0.0, dbLockContext.saturation(t1), 1E-6); + tableContext.acquire(t1, LockType.IS); + dbLockContext.escalate(t1); + assertEquals(0.0, dbLockContext.saturation(t1), 1E-6); + } + +} diff --git a/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockManager.java b/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockManager.java new file mode 100644 index 0000000..415713f --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockManager.java @@ -0,0 +1,479 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.AbstractTransactionContext; +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.Pair; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.*; + +@Category({HW4Tests.class, HW4Part1Tests.class}) +public class TestLockManager { + private LoggingLockManager lockman; + private TransactionContext[] transactions; + private ResourceName dbResource; + private ResourceName[] tables; + + // 2 seconds per test + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 2000 * TimeoutScaling.factor))); + + static boolean holds(LockManager lockman, TransactionContext transaction, ResourceName name, + LockType type) { + List locks = lockman.getLocks(transaction); + if (locks == null) { + return false; + } + for (Lock lock : locks) { + if (lock.name == name && lock.lockType == type) { + return true; + } + } + return false; + } + + @Before + public void setUp() { + lockman = new LoggingLockManager(); + transactions = new TransactionContext[8]; + dbResource = new ResourceName(new Pair<>("database", 0L)); + tables = new ResourceName[transactions.length]; + for (int i = 0; i < transactions.length; ++i) { + transactions[i] = new DummyTransactionContext(lockman, i); + tables[i] = new ResourceName(dbResource, new Pair<>("table" + i, (long) i)); + } + } + + @Test + @Category(PublicTests.class) + public void testSimpleAcquireRelease() { + DeterministicRunner runner = new DeterministicRunner(1); + + runner.run(0, () -> { + lockman.acquireAndRelease(transactions[0], tables[0], LockType.S, Collections.emptyList()); + lockman.acquireAndRelease(transactions[0], tables[1], LockType.S, Collections.singletonList(tables[0])); + }); + assertEquals(LockType.NL, lockman.getLockType(transactions[0], tables[0])); + assertEquals(Collections.emptyList(), lockman.getLocks(tables[0])); + assertEquals(LockType.S, lockman.getLockType(transactions[0], tables[1])); + assertEquals(Collections.singletonList(new Lock(tables[1], LockType.S, 0L)), + lockman.getLocks(tables[1])); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testAcquireReleaseQueue() { + DeterministicRunner runner = new DeterministicRunner(2); + + runner.run(0, () -> lockman.acquireAndRelease(transactions[0], tables[0], LockType.X, + Collections.emptyList())); + runner.run(1, () -> lockman.acquireAndRelease(transactions[1], tables[1], LockType.X, + Collections.emptyList())); + runner.run(0, () -> lockman.acquireAndRelease(transactions[0], tables[1], LockType.X, + Collections.singletonList(tables[0]))); + assertEquals(LockType.X, lockman.getLockType(transactions[0], tables[0])); + assertEquals(Collections.singletonList(new Lock(tables[0], LockType.X, 0L)), + lockman.getLocks(tables[0])); + assertEquals(LockType.NL, lockman.getLockType(transactions[0], tables[1])); + assertEquals(Collections.singletonList(new Lock(tables[1], LockType.X, 1L)), + lockman.getLocks(tables[1])); + assertTrue(transactions[0].getBlocked()); + + runner.join(1); + } + + @Test + @Category(PublicTests.class) + public void testAcquireReleaseDuplicateLock() { + DeterministicRunner runner = new DeterministicRunner(1); + + runner.run(0, () -> lockman.acquireAndRelease(transactions[0], tables[0], LockType.X, + Collections.emptyList())); + try { + runner.run(0, () -> lockman.acquireAndRelease(transactions[0], tables[0], LockType.X, + Collections.emptyList())); + fail(); + } catch (DuplicateLockRequestException e) { + // do nothing + } + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testAcquireReleaseNotHeld() { + DeterministicRunner runner = new DeterministicRunner(1); + + runner.run(0, () -> lockman.acquireAndRelease(transactions[0], tables[0], LockType.X, + Collections.emptyList())); + try { + runner.run(0, () -> + lockman.acquireAndRelease(transactions[0], tables[2], LockType.X, Arrays.asList(tables[0], + tables[1]))); + fail(); + } catch (NoLockHeldException e) { + // do nothing + } + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testAcquireReleaseUpgrade() { + DeterministicRunner runner = new DeterministicRunner(1); + + runner.run(0, () -> { + lockman.acquireAndRelease(transactions[0], tables[0], LockType.S, Collections.emptyList()); + lockman.acquireAndRelease(transactions[0], tables[0], LockType.X, + Collections.singletonList(tables[0])); + }); + assertEquals(LockType.X, lockman.getLockType(transactions[0], tables[0])); + assertEquals(Collections.singletonList(new Lock(tables[0], LockType.X, 0L)), + lockman.getLocks(tables[0])); + assertFalse(transactions[0].getBlocked()); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSimpleAcquireLock() { + DeterministicRunner runner = new DeterministicRunner(2); + + runner.run(0, () -> lockman.acquire(transactions[0], tables[0], LockType.S)); + runner.run(1, () -> lockman.acquire(transactions[1], tables[1], LockType.X)); + assertEquals(LockType.S, lockman.getLockType(transactions[0], tables[0])); + assertEquals(Collections.singletonList(new Lock(tables[0], LockType.S, 0L)), + lockman.getLocks(tables[0])); + assertEquals(LockType.X, lockman.getLockType(transactions[1], tables[1])); + assertEquals(Collections.singletonList(new Lock(tables[1], LockType.X, 1L)), + lockman.getLocks(tables[1])); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSimpleAcquireLockFail() { + DeterministicRunner runner = new DeterministicRunner(1); + + TransactionContext t1 = transactions[0]; + ResourceName r1 = dbResource; + + runner.run(0, () -> lockman.acquire(t1, r1, LockType.X)); + try { + runner.run(0, () -> lockman.acquire(t1, r1, LockType.X)); + fail(); + } catch (DuplicateLockRequestException e) { + // do nothing + } + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSimpleReleaseLock() { + DeterministicRunner runner = new DeterministicRunner(1); + + runner.run(0, () -> { + lockman.acquire(transactions[0], dbResource, LockType.X); + lockman.release(transactions[0], dbResource); + }); + assertEquals(LockType.NL, lockman.getLockType(transactions[0], dbResource)); + assertEquals(Collections.emptyList(), lockman.getLocks(dbResource)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSimpleConflict() { + DeterministicRunner runner = new DeterministicRunner(2); + + runner.run(0, () -> lockman.acquire(transactions[0], dbResource, LockType.X)); + runner.run(1, () -> lockman.acquire(transactions[1], dbResource, LockType.X)); + assertEquals(LockType.X, lockman.getLockType(transactions[0], dbResource)); + assertEquals(LockType.NL, lockman.getLockType(transactions[1], dbResource)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.X, 0L)), + lockman.getLocks(dbResource)); + assertFalse(transactions[0].getBlocked()); + assertTrue(transactions[1].getBlocked()); + + runner.run(0, () -> lockman.release(transactions[0], dbResource)); + assertEquals(LockType.NL, lockman.getLockType(transactions[0], dbResource)); + assertEquals(LockType.X, lockman.getLockType(transactions[1], dbResource)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.X, 1L)), + lockman.getLocks(dbResource)); + assertFalse(transactions[0].getBlocked()); + assertFalse(transactions[1].getBlocked()); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSXS() { + DeterministicRunner runner = new DeterministicRunner(3); + List blocked_status = new ArrayList<>(); + + runner.run(0, () -> lockman.acquire(transactions[0], dbResource, LockType.S)); + runner.run(1, () -> lockman.acquire(transactions[1], dbResource, LockType.X)); + runner.run(2, () -> lockman.acquire(transactions[2], dbResource, LockType.S)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.S, 0L)), + lockman.getLocks(dbResource)); + for (int i = 0; i < 3; ++i) { blocked_status.add(i, transactions[i].getBlocked()); } + assertEquals(Arrays.asList(false, true, true), blocked_status); + + runner.run(0, () -> lockman.release(transactions[0], dbResource)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.X, 1L)), + lockman.getLocks(dbResource)); + blocked_status.clear(); + for (int i = 0; i < 3; ++i) { blocked_status.add(i, transactions[i].getBlocked()); } + assertEquals(Arrays.asList(false, false, true), blocked_status); + + runner.run(1, () -> lockman.release(transactions[1], dbResource)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.S, 2L)), + lockman.getLocks(dbResource)); + blocked_status.clear(); + for (int i = 0; i < 3; ++i) { blocked_status.add(i, transactions[i].getBlocked()); } + assertEquals(Arrays.asList(false, false, false), blocked_status); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testXSXS() { + DeterministicRunner runner = new DeterministicRunner(4); + List blocked_status = new ArrayList<>(); + + runner.run(0, () -> lockman.acquire(transactions[0], dbResource, LockType.X)); + runner.run(1, () -> lockman.acquire(transactions[1], dbResource, LockType.S)); + runner.run(2, () -> lockman.acquire(transactions[2], dbResource, LockType.X)); + runner.run(3, () -> lockman.acquire(transactions[3], dbResource, LockType.S)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.X, 0L)), + lockman.getLocks(dbResource)); + for (int i = 0; i < 4; ++i) { blocked_status.add(i, transactions[i].getBlocked()); } + assertEquals(Arrays.asList(false, true, true, true), blocked_status); + + runner.run(0, () -> lockman.release(transactions[0], dbResource)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.S, 1L)), + lockman.getLocks(dbResource)); + blocked_status.clear(); + for (int i = 0; i < 4; ++i) { blocked_status.add(i, transactions[i].getBlocked()); } + assertEquals(Arrays.asList(false, false, true, true), blocked_status); + + runner.run(1, () -> lockman.release(transactions[1], dbResource)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.X, 2L)), + lockman.getLocks(dbResource)); + blocked_status.clear(); + for (int i = 0; i < 4; ++i) { blocked_status.add(i, transactions[i].getBlocked()); } + assertEquals(Arrays.asList(false, false, false, true), blocked_status); + + runner.run(2, () -> lockman.release(transactions[2], dbResource)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.S, 3L)), + lockman.getLocks(dbResource)); + blocked_status.clear(); + for (int i = 0; i < 4; ++i) { blocked_status.add(i, transactions[i].getBlocked()); } + assertEquals(Arrays.asList(false, false, false, false), blocked_status); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSimplePromoteLock() { + DeterministicRunner runner = new DeterministicRunner(1); + + runner.run(0, () -> { + lockman.acquire(transactions[0], dbResource, LockType.S); + lockman.promote(transactions[0], dbResource, LockType.X); + }); + assertEquals(LockType.X, lockman.getLockType(transactions[0], dbResource)); + assertEquals(Collections.singletonList(new Lock(dbResource, LockType.X, 0L)), + lockman.getLocks(dbResource)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSimplePromoteLockNotHeld() { + DeterministicRunner runner = new DeterministicRunner(1); + try { + runner.run(0, () -> lockman.promote(transactions[0], dbResource, LockType.X)); + fail(); + } catch (NoLockHeldException e) { + // do nothing + } + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testSimplePromoteLockAlreadyHeld() { + DeterministicRunner runner = new DeterministicRunner(1); + + runner.run(0, () -> lockman.acquire(transactions[0], dbResource, LockType.X)); + try { + runner.run(0, () -> lockman.promote(transactions[0], dbResource, LockType.X)); + fail(); + } catch (DuplicateLockRequestException e) { + // do nothing + } + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testFIFOQueueLocks() { + DeterministicRunner runner = new DeterministicRunner(3); + + runner.run(0, () -> lockman.acquire(transactions[0], dbResource, LockType.X)); + runner.run(1, () -> lockman.acquire(transactions[1], dbResource, LockType.X)); + runner.run(2, () -> lockman.acquire(transactions[2], dbResource, LockType.X)); + + assertTrue(holds(lockman, transactions[0], dbResource, LockType.X)); + assertFalse(holds(lockman, transactions[1], dbResource, LockType.X)); + assertFalse(holds(lockman, transactions[2], dbResource, LockType.X)); + + runner.run(0, () -> lockman.release(transactions[0], dbResource)); + + assertFalse(holds(lockman, transactions[0], dbResource, LockType.X)); + assertTrue(holds(lockman, transactions[1], dbResource, LockType.X)); + assertFalse(holds(lockman, transactions[2], dbResource, LockType.X)); + + runner.run(1, () -> lockman.release(transactions[1], dbResource)); + + assertFalse(holds(lockman, transactions[0], dbResource, LockType.X)); + assertFalse(holds(lockman, transactions[1], dbResource, LockType.X)); + assertTrue(holds(lockman, transactions[2], dbResource, LockType.X)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testStatusUpdates() { + DeterministicRunner runner = new DeterministicRunner(2); + + TransactionContext t1 = transactions[0]; + TransactionContext t2 = transactions[1]; + + ResourceName r1 = dbResource; + + runner.run(0, () -> lockman.acquire(t1, r1, LockType.X)); + runner.run(1, () -> lockman.acquire(t2, r1, LockType.X)); + + assertTrue(holds(lockman, t1, r1, LockType.X)); + assertFalse(holds(lockman, t2, r1, LockType.X)); + assertFalse(t1.getBlocked()); + assertTrue(t2.getBlocked()); + + runner.run(0, () -> lockman.release(t1, r1)); + + assertFalse(holds(lockman, t1, r1, LockType.X)); + assertTrue(holds(lockman, t2, r1, LockType.X)); + assertFalse(t1.getBlocked()); + assertFalse(t2.getBlocked()); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testTableEventualUpgrade() { + DeterministicRunner runner = new DeterministicRunner(2); + + TransactionContext t1 = transactions[0]; + TransactionContext t2 = transactions[1]; + + ResourceName r1 = dbResource; + + runner.run(0, () -> lockman.acquire(t1, r1, LockType.S)); + runner.run(1, () -> lockman.acquire(t2, r1, LockType.S)); + + assertTrue(holds(lockman, t1, r1, LockType.S)); + assertTrue(holds(lockman, t2, r1, LockType.S)); + + runner.run(0, () -> lockman.promote(t1, r1, LockType.X)); + + assertTrue(holds(lockman, t1, r1, LockType.S)); + assertFalse(holds(lockman, t1, r1, LockType.X)); + assertTrue(holds(lockman, t2, r1, LockType.S)); + + runner.run(1, () -> lockman.release(t2, r1)); + + assertTrue(holds(lockman, t1, r1, LockType.X)); + assertFalse(holds(lockman, t2, r1, LockType.S)); + + runner.run(0, () -> lockman.release(t1, r1)); + + assertFalse(holds(lockman, t1, r1, LockType.X)); + assertFalse(holds(lockman, t2, r1, LockType.S)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testIntentBlockedAcquire() { + DeterministicRunner runner = new DeterministicRunner(2); + + TransactionContext t1 = transactions[0]; + TransactionContext t2 = transactions[1]; + + ResourceName r0 = dbResource; + + runner.run(0, () -> lockman.acquire(t1, r0, LockType.S)); + runner.run(1, () -> lockman.acquire(t2, r0, LockType.IX)); + + assertTrue(holds(lockman, t1, r0, LockType.S)); + assertFalse(holds(lockman, t2, r0, LockType.IX)); + + runner.run(0, () -> lockman.release(t1, r0)); + + assertFalse(holds(lockman, t1, r0, LockType.S)); + assertTrue(holds(lockman, t2, r0, LockType.IX)); + + runner.joinAll(); + } + + @Test + @Category(PublicTests.class) + public void testReleaseUnheldLock() { + DeterministicRunner runner = new DeterministicRunner(1); + + TransactionContext t1 = transactions[0]; + try { + runner.run(0, () -> lockman.release(t1, dbResource)); + fail(); + } catch (NoLockHeldException e) { + // do nothing + } + + runner.joinAll(); + } + +} + diff --git a/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockType.java b/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockType.java new file mode 100644 index 0000000..0f4a2ea --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockType.java @@ -0,0 +1,95 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.categories.*; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +@Category({HW4Tests.class, HW4Part1Tests.class}) +public class TestLockType { + // 200ms per test + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 200 * TimeoutScaling.factor))); + + @Test + @Category(PublicTests.class) + public void testCompatibleNL() { + assertTrue(LockType.compatible(LockType.NL, LockType.NL)); + assertTrue(LockType.compatible(LockType.NL, LockType.S)); + assertTrue(LockType.compatible(LockType.NL, LockType.X)); + assertTrue(LockType.compatible(LockType.NL, LockType.IS)); + assertTrue(LockType.compatible(LockType.NL, LockType.IX)); + assertTrue(LockType.compatible(LockType.NL, LockType.SIX)); + assertTrue(LockType.compatible(LockType.S, LockType.NL)); + assertTrue(LockType.compatible(LockType.X, LockType.NL)); + assertTrue(LockType.compatible(LockType.IS, LockType.NL)); + assertTrue(LockType.compatible(LockType.IX, LockType.NL)); + assertTrue(LockType.compatible(LockType.SIX, LockType.NL)); + } + + @Test + @Category(PublicTests.class) + public void testCompatibleS() { + assertTrue(LockType.compatible(LockType.S, LockType.S)); + assertFalse(LockType.compatible(LockType.S, LockType.X)); + assertTrue(LockType.compatible(LockType.S, LockType.IS)); + assertFalse(LockType.compatible(LockType.S, LockType.IX)); + assertFalse(LockType.compatible(LockType.S, LockType.SIX)); + assertFalse(LockType.compatible(LockType.X, LockType.S)); + assertTrue(LockType.compatible(LockType.IS, LockType.S)); + assertFalse(LockType.compatible(LockType.IX, LockType.S)); + assertFalse(LockType.compatible(LockType.SIX, LockType.S)); + } + + @Test + @Category(SystemTests.class) + public void testParent() { + assertEquals(LockType.NL, LockType.parentLock(LockType.NL)); + assertEquals(LockType.IS, LockType.parentLock(LockType.S)); + assertEquals(LockType.IX, LockType.parentLock(LockType.X)); + assertEquals(LockType.IS, LockType.parentLock(LockType.IS)); + assertEquals(LockType.IX, LockType.parentLock(LockType.IX)); + assertEquals(LockType.IX, LockType.parentLock(LockType.SIX)); + } + + @Test + @Category(PublicTests.class) + public void testCanBeParentNL() { + for (LockType lockType : LockType.values()) { + assertTrue(LockType.canBeParentLock(lockType, LockType.NL)); + } + + for (LockType lockType : LockType.values()) { + if (lockType != LockType.NL) { + assertFalse(LockType.canBeParentLock(LockType.NL, lockType)); + } + } + } + + @Test + @Category(PublicTests.class) + public void testSubstitutableReal() { + assertTrue(LockType.substitutable(LockType.S, LockType.S)); + assertTrue(LockType.substitutable(LockType.X, LockType.S)); + assertFalse(LockType.substitutable(LockType.IS, LockType.S)); + assertFalse(LockType.substitutable(LockType.IX, LockType.S)); + assertTrue(LockType.substitutable(LockType.SIX, LockType.S)); + assertFalse(LockType.substitutable(LockType.S, LockType.X)); + assertTrue(LockType.substitutable(LockType.X, LockType.X)); + assertFalse(LockType.substitutable(LockType.IS, LockType.X)); + assertFalse(LockType.substitutable(LockType.IX, LockType.X)); + assertFalse(LockType.substitutable(LockType.SIX, LockType.X)); + } + +} + diff --git a/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockUtil.java b/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockUtil.java new file mode 100644 index 0000000..71d1cb6 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/concurrency/TestLockUtil.java @@ -0,0 +1,150 @@ +package edu.berkeley.cs186.database.concurrency; + +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.categories.HW4Part2Tests; +import edu.berkeley.cs186.database.categories.HW4Tests; +import edu.berkeley.cs186.database.categories.HiddenTests; +import edu.berkeley.cs186.database.categories.PublicTests; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category({HW4Tests.class, HW4Part2Tests.class}) +public class TestLockUtil { + private LoggingLockManager lockManager; + private TransactionContext transaction; + private LockContext dbContext; + private LockContext tableContext; + private LockContext[] pageContexts; + + // 1 second per test + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + @Before + public void setUp() { + lockManager = new LoggingLockManager(); + transaction = new DummyTransactionContext(lockManager, 0); + dbContext = lockManager.databaseContext(); + tableContext = dbContext.childContext("table1", 1); + pageContexts = new LockContext[8]; + for (int i = 0; i < pageContexts.length; ++i) { + pageContexts[i] = tableContext.childContext((long) i); + } + TransactionContext.setTransaction(transaction); + } + + @Test + @Category(PublicTests.class) + public void testRequestNullTransaction() { + lockManager.startLog(); + TransactionContext.setTransaction(null); + LockUtil.ensureSufficientLockHeld(pageContexts[4], LockType.S); + assertEquals(Collections.emptyList(), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testSimpleAcquire() { + lockManager.startLog(); + LockUtil.ensureSufficientLockHeld(pageContexts[4], LockType.S); + assertEquals(Arrays.asList( + "acquire 0 database IS", + "acquire 0 database/table1 IS", + "acquire 0 database/table1/4 S" + ), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testSimplePromote() { + LockUtil.ensureSufficientLockHeld(pageContexts[4], LockType.S); + lockManager.startLog(); + LockUtil.ensureSufficientLockHeld(pageContexts[4], LockType.X); + assertEquals(Arrays.asList( + "promote 0 database IX", + "promote 0 database/table1 IX", + "promote 0 database/table1/4 X" + ), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testIStoS() { + LockUtil.ensureSufficientLockHeld(pageContexts[4], LockType.S); + pageContexts[4].release(transaction); + lockManager.startLog(); + LockUtil.ensureSufficientLockHeld(tableContext, LockType.S); + assertEquals(Collections.singletonList( + "acquire-and-release 0 database/table1 S [database/table1]" + ), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testSimpleEscalate() { + LockUtil.ensureSufficientLockHeld(pageContexts[4], LockType.S); + lockManager.startLog(); + LockUtil.ensureSufficientLockHeld(tableContext, LockType.S); + assertEquals(Collections.singletonList( + "acquire-and-release 0 database/table1 S [database/table1, database/table1/4]" + ), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testIXBeforeIS() { + LockUtil.ensureSufficientLockHeld(pageContexts[3], LockType.X); + lockManager.startLog(); + LockUtil.ensureSufficientLockHeld(pageContexts[4], LockType.S); + assertEquals(Collections.singletonList( + "acquire 0 database/table1/4 S" + ), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testSIX1() { + LockUtil.ensureSufficientLockHeld(pageContexts[3], LockType.X); + lockManager.startLog(); + LockUtil.ensureSufficientLockHeld(tableContext, LockType.S); + assertEquals(Collections.singletonList( + "acquire-and-release 0 database/table1 SIX [database/table1]" + ), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testSIX2() { + LockUtil.ensureSufficientLockHeld(pageContexts[1], LockType.S); + LockUtil.ensureSufficientLockHeld(pageContexts[2], LockType.S); + LockUtil.ensureSufficientLockHeld(pageContexts[3], LockType.X); + lockManager.startLog(); + LockUtil.ensureSufficientLockHeld(tableContext, LockType.S); + assertEquals(Collections.singletonList( + "acquire-and-release 0 database/table1 SIX [database/table1, database/table1/1, database/table1/2]" + ), lockManager.log); + } + + @Test + @Category(PublicTests.class) + public void testSimpleNL() { + lockManager.startLog(); + LockUtil.ensureSufficientLockHeld(tableContext, LockType.NL); + assertEquals(Collections.emptyList(), lockManager.log); + } + +} + diff --git a/src/test/java/edu/berkeley/cs186/database/databox/TestBoolDataBox.java b/src/test/java/edu/berkeley/cs186/database/databox/TestBoolDataBox.java new file mode 100644 index 0000000..74482eb --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/databox/TestBoolDataBox.java @@ -0,0 +1,75 @@ +package edu.berkeley.cs186.database.databox; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestBoolDataBox { + @Test + public void testType() { + assertEquals(Type.boolType(), new BoolDataBox(true).type()); + assertEquals(Type.boolType(), new BoolDataBox(false).type()); + } + + @Test + public void testGetBool() { + assertEquals(true, new BoolDataBox(true).getBool()); + assertEquals(false, new BoolDataBox(false).getBool()); + } + + @Test(expected = DataBoxException.class) + public void testGetInt() { + new BoolDataBox(true).getInt(); + } + + @Test(expected = DataBoxException.class) + public void testGetLong() { + new BoolDataBox(true).getLong(); + } + + @Test(expected = DataBoxException.class) + public void testGetFloat() { + new BoolDataBox(true).getFloat(); + } + + @Test(expected = DataBoxException.class) + public void testGetString() { + new BoolDataBox(true).getString(); + } + + @Test + public void testToAndFromBytes() { + for (boolean b : new boolean[] {true, false}) { + BoolDataBox d = new BoolDataBox(b); + byte[] bytes = d.toBytes(); + assertEquals(d, DataBox.fromBytes(ByteBuffer.wrap(bytes), Type.boolType())); + } + } + + @Test + public void testEquals() { + BoolDataBox tru = new BoolDataBox(true); + BoolDataBox fls = new BoolDataBox(false); + assertEquals(tru, tru); + assertEquals(fls, fls); + assertNotEquals(tru, fls); + assertNotEquals(fls, tru); + } + + @Test + public void testCompareTo() { + BoolDataBox tru = new BoolDataBox(true); + BoolDataBox fls = new BoolDataBox(false); + assertTrue(fls.compareTo(fls) == 0); + assertTrue(fls.compareTo(tru) < 0); + assertTrue(tru.compareTo(tru) == 0); + assertTrue(tru.compareTo(fls) > 0); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/databox/TestFloatDataBox.java b/src/test/java/edu/berkeley/cs186/database/databox/TestFloatDataBox.java new file mode 100644 index 0000000..3c49191 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/databox/TestFloatDataBox.java @@ -0,0 +1,73 @@ +package edu.berkeley.cs186.database.databox; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestFloatDataBox { + @Test + public void testType() { + assertEquals(Type.floatType(), new FloatDataBox(0f).type()); + } + + @Test(expected = DataBoxException.class) + public void testGetBool() { + new FloatDataBox(0f).getBool(); + } + + @Test(expected = DataBoxException.class) + public void testGetInt() { + new FloatDataBox(0f).getInt(); + } + + @Test(expected = DataBoxException.class) + public void testGetLong() { + new FloatDataBox(0f).getLong(); + } + + @Test + public void testGetFloat() { + assertEquals(0f, new FloatDataBox(0f).getFloat(), 0.0001); + } + + @Test(expected = DataBoxException.class) + public void testGetString() { + new FloatDataBox(0f).getString(); + } + + @Test + public void testToAndFromBytes() { + for (int i = -10; i < 10; ++i) { + FloatDataBox d = new FloatDataBox((float) i); + byte[] bytes = d.toBytes(); + assertEquals(d, DataBox.fromBytes(ByteBuffer.wrap(bytes), Type.floatType())); + } + } + + @Test + public void testEquals() { + FloatDataBox zero = new FloatDataBox(0f); + FloatDataBox one = new FloatDataBox(1f); + assertEquals(zero, zero); + assertEquals(one, one); + assertNotEquals(zero, one); + assertNotEquals(one, zero); + } + + @Test + public void testCompareTo() { + FloatDataBox zero = new FloatDataBox(0f); + FloatDataBox one = new FloatDataBox(1f); + assertTrue(zero.compareTo(zero) == 0); + assertTrue(zero.compareTo(one) < 0); + assertTrue(one.compareTo(one) == 0); + assertTrue(one.compareTo(zero) > 0); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/databox/TestIntDataBox.java b/src/test/java/edu/berkeley/cs186/database/databox/TestIntDataBox.java new file mode 100644 index 0000000..ac1fc3b --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/databox/TestIntDataBox.java @@ -0,0 +1,73 @@ +package edu.berkeley.cs186.database.databox; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestIntDataBox { + @Test + public void testType() { + assertEquals(Type.intType(), new IntDataBox(0).type()); + } + + @Test(expected = DataBoxException.class) + public void testGetBool() { + new IntDataBox(0).getBool(); + } + + @Test + public void testGetInt() { + assertEquals(0, new IntDataBox(0).getInt()); + } + + @Test(expected = DataBoxException.class) + public void testGetLong() { + new IntDataBox(0).getLong(); + } + + @Test(expected = DataBoxException.class) + public void testGetFloat() { + new IntDataBox(0).getFloat(); + } + + @Test(expected = DataBoxException.class) + public void testGetString() { + new IntDataBox(0).getString(); + } + + @Test + public void testToAndFromBytes() { + for (int i = -10; i < 10; ++i) { + IntDataBox d = new IntDataBox(i); + byte[] bytes = d.toBytes(); + assertEquals(d, DataBox.fromBytes(ByteBuffer.wrap(bytes), Type.intType())); + } + } + + @Test + public void testEquals() { + IntDataBox zero = new IntDataBox(0); + IntDataBox one = new IntDataBox(1); + assertEquals(zero, zero); + assertEquals(one, one); + assertNotEquals(zero, one); + assertNotEquals(one, zero); + } + + @Test + public void testCompareTo() { + IntDataBox zero = new IntDataBox(0); + IntDataBox one = new IntDataBox(1); + assertTrue(zero.compareTo(zero) == 0); + assertTrue(zero.compareTo(one) < 0); + assertTrue(one.compareTo(one) == 0); + assertTrue(one.compareTo(zero) > 0); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/databox/TestLongDataBox.java b/src/test/java/edu/berkeley/cs186/database/databox/TestLongDataBox.java new file mode 100644 index 0000000..9395161 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/databox/TestLongDataBox.java @@ -0,0 +1,71 @@ +package edu.berkeley.cs186.database.databox; + +import edu.berkeley.cs186.database.categories.HW99Tests; +import edu.berkeley.cs186.database.categories.SystemTests; +import edu.berkeley.cs186.database.common.ByteBuffer; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.*; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestLongDataBox { + @Test + public void testType() { + assertEquals(Type.longType(), new LongDataBox(0L).type()); + } + + @Test(expected = DataBoxException.class) + public void testGetBool() { + new LongDataBox(0L).getBool(); + } + + @Test(expected = DataBoxException.class) + public void testGetInt() { + new LongDataBox(0L).getInt(); + } + + @Test + public void testGetLong() { + assertEquals(0L, new LongDataBox(0L).getLong()); + } + + @Test(expected = DataBoxException.class) + public void testGetFloat() { + new LongDataBox(0L).getFloat(); + } + + @Test(expected = DataBoxException.class) + public void testGetString() { + new LongDataBox(0L).getString(); + } + + @Test + public void testToAndFromBytes() { + for (long i = -10L; i < 10L; ++i) { + LongDataBox d = new LongDataBox(i); + byte[] bytes = d.toBytes(); + assertEquals(d, DataBox.fromBytes(ByteBuffer.wrap(bytes), Type.longType())); + } + } + + @Test + public void testEquals() { + LongDataBox zero = new LongDataBox(0L); + LongDataBox one = new LongDataBox(1L); + assertEquals(zero, zero); + assertEquals(one, one); + assertNotEquals(zero, one); + assertNotEquals(one, zero); + } + + @Test + public void testCompareTo() { + LongDataBox zero = new LongDataBox(0L); + LongDataBox one = new LongDataBox(1L); + assertTrue(zero.compareTo(zero) == 0L); + assertTrue(zero.compareTo(one) < 0L); + assertTrue(one.compareTo(one) == 0L); + assertTrue(one.compareTo(zero) > 0L); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/databox/TestStringDataBox.java b/src/test/java/edu/berkeley/cs186/database/databox/TestStringDataBox.java new file mode 100644 index 0000000..087e9eb --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/databox/TestStringDataBox.java @@ -0,0 +1,82 @@ +package edu.berkeley.cs186.database.databox; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestStringDataBox { + @Test(expected = DataBoxException.class) + public void testEmptyString() { + new StringDataBox("", 0); + } + + @Test + public void testType() { + assertEquals(Type.stringType(3), new StringDataBox("foo", 3).type()); + } + + @Test(expected = DataBoxException.class) + public void testGetBool() { + new StringDataBox("foo", 3).getBool(); + } + + @Test(expected = DataBoxException.class) + public void testGetInt() { + new StringDataBox("foo", 3).getInt(); + } + + @Test(expected = DataBoxException.class) + public void testGetLong() { + new StringDataBox("foo", 3).getLong(); + } + + @Test(expected = DataBoxException.class) + public void testGetFloat() { + new StringDataBox("foo", 3).getFloat(); + } + + @Test + public void testGetString() { + assertEquals("f", new StringDataBox("foo", 1).getString()); + assertEquals("fo", new StringDataBox("foo", 2).getString()); + assertEquals("foo", new StringDataBox("foo", 3).getString()); + assertEquals("foo", new StringDataBox("foo", 4).getString()); + assertEquals("foo", new StringDataBox("foo", 5).getString()); + } + + @Test + public void testToAndFromBytes() { + for (String s : new String[] {"foo", "bar", "baz"}) { + StringDataBox d = new StringDataBox(s, 3); + byte[] bytes = d.toBytes(); + assertEquals(d, DataBox.fromBytes(ByteBuffer.wrap(bytes), Type.stringType(3))); + } + } + + @Test + public void testEquals() { + StringDataBox foo = new StringDataBox("foo", 3); + StringDataBox zoo = new StringDataBox("zoo", 3); + assertEquals(foo, foo); + assertEquals(zoo, zoo); + assertNotEquals(foo, zoo); + assertNotEquals(zoo, foo); + } + + @Test + public void testCompareTo() { + StringDataBox foo = new StringDataBox("foo", 3); + StringDataBox zoo = new StringDataBox("zoo", 3); + assertTrue(foo.compareTo(foo) == 0); + assertTrue(foo.compareTo(zoo) < 0); + assertTrue(zoo.compareTo(zoo) == 0); + assertTrue(zoo.compareTo(foo) > 0); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/databox/TestType.java b/src/test/java/edu/berkeley/cs186/database/databox/TestType.java new file mode 100644 index 0000000..dfe3c6d --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/databox/TestType.java @@ -0,0 +1,114 @@ +package edu.berkeley.cs186.database.databox; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.Buffer; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestType { + @Test + public void testBoolType() { + // Check type id and size. + Type boolType = Type.boolType(); + assertEquals(boolType.getTypeId(), TypeId.BOOL); + assertEquals(boolType.getSizeInBytes(), 1); + + // Check toBytes and fromBytes. + Buffer buf = ByteBuffer.wrap(boolType.toBytes()); + assertEquals(boolType, Type.fromBytes(buf)); + + // Check equality. + assertEquals(boolType, Type.boolType()); + assertNotEquals(boolType, Type.intType()); + assertNotEquals(boolType, Type.floatType()); + assertNotEquals(boolType, Type.stringType(1)); + assertNotEquals(boolType, Type.stringType(2)); + } + + @Test + public void testIntType() { + // Check type id and size. + Type intType = Type.intType(); + assertEquals(intType.getTypeId(), TypeId.INT); + assertEquals(intType.getSizeInBytes(), 4); + + // Check toBytes and fromBytes. + Buffer buf = ByteBuffer.wrap(intType.toBytes()); + assertEquals(intType, Type.fromBytes(buf)); + + // Check equality. + assertNotEquals(intType, Type.boolType()); + assertEquals(intType, Type.intType()); + assertNotEquals(intType, Type.floatType()); + assertNotEquals(intType, Type.stringType(1)); + assertNotEquals(intType, Type.stringType(2)); + } + + @Test + public void testFloatType() { + // Check type id and size. + Type floatType = Type.floatType(); + assertEquals(floatType.getTypeId(), TypeId.FLOAT); + assertEquals(floatType.getSizeInBytes(), 4); + + // Check toBytes and fromBytes. + Buffer buf = ByteBuffer.wrap(floatType.toBytes()); + assertEquals(floatType, Type.fromBytes(buf)); + + // Check equality. + assertNotEquals(floatType, Type.boolType()); + assertNotEquals(floatType, Type.intType()); + assertEquals(floatType, Type.floatType()); + assertNotEquals(floatType, Type.stringType(1)); + assertNotEquals(floatType, Type.stringType(2)); + } + + @Test(expected = DataBoxException.class) + public void testZeroByteStringype() { + Type.stringType(0); + } + + @Test + public void testOneByteStringype() { + // Check type id and size. + Type stringType = Type.stringType(1); + assertEquals(stringType.getTypeId(), TypeId.STRING); + assertEquals(stringType.getSizeInBytes(), 1); + + // Check toBytes and fromBytes. + Buffer buf = ByteBuffer.wrap(stringType.toBytes()); + assertEquals(stringType, Type.fromBytes(buf)); + + // Check equality. + assertNotEquals(stringType, Type.boolType()); + assertNotEquals(stringType, Type.intType()); + assertNotEquals(stringType, Type.floatType()); + assertEquals(stringType, Type.stringType(1)); + assertNotEquals(stringType, Type.stringType(2)); + } + + @Test + public void testTwoByteStringype() { + // Check type id and size. + Type stringType = Type.stringType(2); + assertEquals(stringType.getTypeId(), TypeId.STRING); + assertEquals(stringType.getSizeInBytes(), 2); + + // Check toBytes and fromBytes. + Buffer buf = ByteBuffer.wrap(stringType.toBytes()); + assertEquals(stringType, Type.fromBytes(buf)); + + // Check equality. + assertNotEquals(stringType, Type.boolType()); + assertNotEquals(stringType, Type.intType()); + assertNotEquals(stringType, Type.floatType()); + assertNotEquals(stringType, Type.stringType(1)); + assertEquals(stringType, Type.stringType(2)); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/databox/TestWelcome.java b/src/test/java/edu/berkeley/cs186/database/databox/TestWelcome.java new file mode 100644 index 0000000..62a9c98 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/databox/TestWelcome.java @@ -0,0 +1,16 @@ +package edu.berkeley.cs186.database.databox; + +import edu.berkeley.cs186.database.categories.HW0Tests; +import edu.berkeley.cs186.database.categories.PublicTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category({HW0Tests.class, PublicTests.class}) +public class TestWelcome { + @Test + public void testComplete() { + assertEquals("welcome", new StringDataBox("welcome", 7).toString()); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/index/TestBPlusNode.java b/src/test/java/edu/berkeley/cs186/database/index/TestBPlusNode.java new file mode 100644 index 0000000..1b7754b --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/index/TestBPlusNode.java @@ -0,0 +1,88 @@ +package edu.berkeley.cs186.database.index; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.MemoryDiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.ClockEvictionPolicy; +import edu.berkeley.cs186.database.recovery.DummyRecoveryManager; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.table.RecordId; + +@Category(HW2Tests.class) +public class TestBPlusNode { + private static final int ORDER = 5; + + private BufferManager bufferManager; + private BPlusTreeMetadata metadata; + private LockContext treeContext; + + // 1 seconds max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + @Before + public void setup() { + DiskSpaceManager diskSpaceManager = new MemoryDiskSpaceManager(); + diskSpaceManager.allocPart(0); + this.bufferManager = new BufferManagerImpl(diskSpaceManager, new DummyRecoveryManager(), 1024, + new ClockEvictionPolicy()); + this.treeContext = new DummyLockContext(); + this.metadata = new BPlusTreeMetadata("test", "col", Type.intType(), ORDER, + 0, DiskSpaceManager.INVALID_PAGE_NUM, -1); + } + + @After + public void cleanup() { + this.bufferManager.close(); + } + + @Test + @Category(PublicTests.class) + public void testFromBytes() { + // Leaf node. + List leafKeys = new ArrayList<>(); + List leafRids = new ArrayList<>(); + for (int i = 0; i < 2 * ORDER; ++i) { + leafKeys.add(new IntDataBox(i)); + leafRids.add(new RecordId(i, (short) i)); + } + LeafNode leaf = new LeafNode(metadata, bufferManager, leafKeys, leafRids, Optional.of(42L), + treeContext); + + // Inner node. + List innerKeys = new ArrayList<>(); + List innerChildren = new ArrayList<>(); + for (int i = 0; i < 2 * ORDER; ++i) { + innerKeys.add(new IntDataBox(i)); + innerChildren.add((long) i); + } + innerChildren.add((long) 2 * ORDER); + InnerNode inner = new InnerNode(metadata, bufferManager, innerKeys, innerChildren, + treeContext); + + long leafPageNum = leaf.getPage().getPageNum(); + long innerPageNum = inner.getPage().getPageNum(); + assertEquals(leaf, BPlusNode.fromBytes(metadata, bufferManager, treeContext, leafPageNum)); + assertEquals(inner, BPlusNode.fromBytes(metadata, bufferManager, treeContext, innerPageNum)); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/index/TestBPlusTree.java b/src/test/java/edu/berkeley/cs186/database/index/TestBPlusTree.java new file mode 100644 index 0000000..f91e42a --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/index/TestBPlusTree.java @@ -0,0 +1,471 @@ +package edu.berkeley.cs186.database.index; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.function.Supplier; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.MemoryDiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.ClockEvictionPolicy; +import edu.berkeley.cs186.database.recovery.DummyRecoveryManager; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.table.RecordId; + +import static org.junit.Assert.*; + +@Category(HW2Tests.class) +public class TestBPlusTree { + private BufferManager bufferManager; + private BPlusTreeMetadata metadata; + private LockContext treeContext; + + // max 5 I/Os per iterator creation default + private static final int MAX_IO_PER_ITER_CREATE = 5; + + // max 1 I/Os per iterator next, unless overridden + private static final int MAX_IO_PER_NEXT = 1; + + // 3 seconds max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 3000 * TimeoutScaling.factor))); + + @Before + public void setup() { + DiskSpaceManager diskSpaceManager = new MemoryDiskSpaceManager(); + diskSpaceManager.allocPart(0); + this.bufferManager = new BufferManagerImpl(diskSpaceManager, new DummyRecoveryManager(), 1024, + new ClockEvictionPolicy()); + this.treeContext = new DummyLockContext(); + this.metadata = null; + } + + @After + public void cleanup() { + this.bufferManager.close(); + } + + // Helpers ///////////////////////////////////////////////////////////////// + private void setBPlusTreeMetadata(Type keySchema, int order) { + this.metadata = new BPlusTreeMetadata("test", "col", keySchema, order, + 0, DiskSpaceManager.INVALID_PAGE_NUM, -1); + } + + private BPlusTree getBPlusTree(Type keySchema, int order) { + setBPlusTreeMetadata(keySchema, order); + return new BPlusTree(bufferManager, metadata, treeContext); + } + + // the 0th item in maxIOsOverride specifies how many I/Os constructing the iterator may take + // the i+1th item in maxIOsOverride specifies how many I/Os the ith call to next() may take + // if there are more items in the iterator than maxIOsOverride, then we default to + // MAX_IO_PER_ITER_CREATE/MAX_IO_PER_NEXT once we run out of items in maxIOsOverride + private List indexIteratorToList(Supplier> iteratorSupplier, + Iterator maxIOsOverride) { + bufferManager.evictAll(); + + long initialIOs = bufferManager.getNumIOs(); + + long prevIOs = initialIOs; + Iterator iter = iteratorSupplier.get(); + long newIOs = bufferManager.getNumIOs(); + long maxIOs = maxIOsOverride.hasNext() ? maxIOsOverride.next() : MAX_IO_PER_ITER_CREATE; + assertFalse("too many I/Os used constructing iterator (" + (newIOs - prevIOs) + " > " + maxIOs + + ") - are you materializing more than you need?", + newIOs - prevIOs > maxIOs); + + List xs = new ArrayList<>(); + while (iter.hasNext()) { + prevIOs = bufferManager.getNumIOs(); + xs.add(iter.next()); + newIOs = bufferManager.getNumIOs(); + maxIOs = maxIOsOverride.hasNext() ? maxIOsOverride.next() : MAX_IO_PER_NEXT; + assertFalse("too many I/Os used per next() call (" + (newIOs - prevIOs) + " > " + maxIOs + + ") - are you materializing more than you need?", + newIOs - prevIOs > maxIOs); + } + + long finalIOs = bufferManager.getNumIOs(); + maxIOs = xs.size() / (2 * metadata.getOrder()); + assertTrue("too few I/Os used overall (" + (finalIOs - initialIOs) + " < " + maxIOs + + ") - are you materializing before the iterator is even constructed?", + (finalIOs - initialIOs) >= maxIOs); + return xs; + } + + private List indexIteratorToList(Supplier> iteratorSupplier) { + return indexIteratorToList(iteratorSupplier, Collections.emptyIterator()); + } + + // Tests /////////////////////////////////////////////////////////////////// + + @Test + @Category(PublicTests.class) + public void testSimpleBulkLoad() { + BPlusTree tree = getBPlusTree(Type.intType(), 2); + float fillFactor = 0.75f; + assertEquals("()", tree.toSexp()); + + List> data = new ArrayList<>(); + for (int i = 1; i <= 11; ++i) { + data.add(new Pair<>(new IntDataBox(i), new RecordId(i, (short) i))); + } + + tree.bulkLoad(data.iterator(), fillFactor); + // ( 4 7 10 _ ) + // / | | \ + // (1 2 3 _) (4 5 6 _) (7 8 9 _) (10 11 _ _) + String leaf0 = "((1 (1 1)) (2 (2 2)) (3 (3 3)))"; + String leaf1 = "((4 (4 4)) (5 (5 5)) (6 (6 6)))"; + String leaf2 = "((7 (7 7)) (8 (8 8)) (9 (9 9)))"; + String leaf3 = "((10 (10 10)) (11 (11 11)))"; + String sexp = String.format("(%s 4 %s 7 %s 10 %s)", leaf0, leaf1, leaf2, leaf3); + assertEquals(sexp, tree.toSexp()); + } + + @Test + @Category(PublicTests.class) + public void testWhiteBoxTest() { + BPlusTree tree = getBPlusTree(Type.intType(), 1); + assertEquals("()", tree.toSexp()); + + // (4) + tree.put(new IntDataBox(4), new RecordId(4, (short) 4)); + assertEquals("((4 (4 4)))", tree.toSexp()); + + // (4 9) + tree.put(new IntDataBox(9), new RecordId(9, (short) 9)); + assertEquals("((4 (4 4)) (9 (9 9)))", tree.toSexp()); + + // (6) + // / \ + // (4) (6 9) + tree.put(new IntDataBox(6), new RecordId(6, (short) 6)); + String l = "((4 (4 4)))"; + String r = "((6 (6 6)) (9 (9 9)))"; + assertEquals(String.format("(%s 6 %s)", l, r), tree.toSexp()); + + // (6) + // / \ + // (2 4) (6 9) + tree.put(new IntDataBox(2), new RecordId(2, (short) 2)); + l = "((2 (2 2)) (4 (4 4)))"; + r = "((6 (6 6)) (9 (9 9)))"; + assertEquals(String.format("(%s 6 %s)", l, r), tree.toSexp()); + + // (6 7) + // / | \ + // (2 4) (6) (7 9) + tree.put(new IntDataBox(7), new RecordId(7, (short) 7)); + l = "((2 (2 2)) (4 (4 4)))"; + String m = "((6 (6 6)))"; + r = "((7 (7 7)) (9 (9 9)))"; + assertEquals(String.format("(%s 6 %s 7 %s)", l, m, r), tree.toSexp()); + + // (7) + // / \ + // (6) (8) + // / \ / \ + // (2 4) (6) (7) (8 9) + tree.put(new IntDataBox(8), new RecordId(8, (short) 8)); + String ll = "((2 (2 2)) (4 (4 4)))"; + String lr = "((6 (6 6)))"; + String rl = "((7 (7 7)))"; + String rr = "((8 (8 8)) (9 (9 9)))"; + l = String.format("(%s 6 %s)", ll, lr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 7 %s)", l, r), tree.toSexp()); + + // (7) + // / \ + // (3 6) (8) + // / | \ / \ + // (2) (3 4) (6) (7) (8 9) + tree.put(new IntDataBox(3), new RecordId(3, (short) 3)); + ll = "((2 (2 2)))"; + String lm = "((3 (3 3)) (4 (4 4)))"; + lr = "((6 (6 6)))"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)) (9 (9 9)))"; + l = String.format("(%s 3 %s 6 %s)", ll, lm, lr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 7 %s)", l, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // (2) (3) (4 5) (6) (7) (8 9) + tree.put(new IntDataBox(5), new RecordId(5, (short) 5)); + ll = "((2 (2 2)))"; + lr = "((3 (3 3)))"; + String ml = "((4 (4 4)) (5 (5 5)))"; + String mr = "((6 (6 6)))"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)) (9 (9 9)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // (1 2) (3) (4 5) (6) (7) (8 9) + tree.put(new IntDataBox(1), new RecordId(1, (short) 1)); + ll = "((1 (1 1)) (2 (2 2)))"; + lr = "((3 (3 3)))"; + ml = "((4 (4 4)) (5 (5 5)))"; + mr = "((6 (6 6)))"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)) (9 (9 9)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( 2) (3) (4 5) (6) (7) (8 9) + tree.remove(new IntDataBox(1)); + ll = "((2 (2 2)))"; + lr = "((3 (3 3)))"; + ml = "((4 (4 4)) (5 (5 5)))"; + mr = "((6 (6 6)))"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)) (9 (9 9)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( 2) (3) (4 5) (6) (7) (8 ) + tree.remove(new IntDataBox(9)); + ll = "((2 (2 2)))"; + lr = "((3 (3 3)))"; + ml = "((4 (4 4)) (5 (5 5)))"; + mr = "((6 (6 6)))"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( 2) (3) (4 5) ( ) (7) (8 ) + tree.remove(new IntDataBox(6)); + ll = "((2 (2 2)))"; + lr = "((3 (3 3)))"; + ml = "((4 (4 4)) (5 (5 5)))"; + mr = "()"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( 2) (3) ( 5) ( ) (7) (8 ) + tree.remove(new IntDataBox(4)); + ll = "((2 (2 2)))"; + lr = "((3 (3 3)))"; + ml = "((5 (5 5)))"; + mr = "()"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( ) (3) ( 5) ( ) (7) (8 ) + tree.remove(new IntDataBox(2)); + ll = "()"; + lr = "((3 (3 3)))"; + ml = "((5 (5 5)))"; + mr = "()"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( ) (3) ( ) ( ) (7) (8 ) + tree.remove(new IntDataBox(5)); + ll = "()"; + lr = "((3 (3 3)))"; + ml = "()"; + mr = "()"; + rl = "((7 (7 7)))"; + rr = "((8 (8 8)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( ) (3) ( ) ( ) ( ) (8 ) + tree.remove(new IntDataBox(7)); + ll = "()"; + lr = "((3 (3 3)))"; + ml = "()"; + mr = "()"; + rl = "()"; + rr = "((8 (8 8)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( ) ( ) ( ) ( ) ( ) (8 ) + tree.remove(new IntDataBox(3)); + ll = "()"; + lr = "()"; + ml = "()"; + mr = "()"; + rl = "()"; + rr = "((8 (8 8)))"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + + // (4 7) + // / | \ + // (3) (6) (8) + // / \ / \ / \ + // ( ) ( ) ( ) ( ) ( ) ( ) + tree.remove(new IntDataBox(8)); + ll = "()"; + lr = "()"; + ml = "()"; + mr = "()"; + rl = "()"; + rr = "()"; + l = String.format("(%s 3 %s)", ll, lr); + m = String.format("(%s 6 %s)", ml, mr); + r = String.format("(%s 8 %s)", rl, rr); + assertEquals(String.format("(%s 4 %s 7 %s)", l, m, r), tree.toSexp()); + } + + @Test + @Category(PublicTests.class) + public void testRandomPuts() { + List keys = new ArrayList<>(); + List rids = new ArrayList<>(); + List sortedRids = new ArrayList<>(); + for (int i = 0; i < 1000; ++i) { + keys.add(new IntDataBox(i)); + rids.add(new RecordId(i, (short) i)); + sortedRids.add(new RecordId(i, (short) i)); + } + + // Try trees with different orders. + for (int d = 2; d < 5; ++d) { + // Try trees with different insertion orders. + for (int n = 0; n < 2; ++n) { + Collections.shuffle(keys, new Random(42)); + Collections.shuffle(rids, new Random(42)); + + // Insert all the keys. + BPlusTree tree = getBPlusTree(Type.intType(), d); + for (int i = 0; i < keys.size(); ++i) { + tree.put(keys.get(i), rids.get(i)); + } + + // Test get. + for (int i = 0; i < keys.size(); ++i) { + assertEquals(Optional.of(rids.get(i)), tree.get(keys.get(i))); + } + + // Test scanAll. + assertEquals(sortedRids, indexIteratorToList(tree::scanAll)); + + // Test scanGreaterEqual. + for (int i = 0; i < keys.size(); i += 100) { + final int j = i; + List expected = sortedRids.subList(i, sortedRids.size()); + assertEquals(expected, indexIteratorToList(() -> tree.scanGreaterEqual(new IntDataBox(j)))); + } + + // Load the tree from disk. + BPlusTree fromDisk = new BPlusTree(bufferManager, metadata, treeContext); + assertEquals(sortedRids, indexIteratorToList(fromDisk::scanAll)); + + // Test remove. + Collections.shuffle(keys, new Random(42)); + Collections.shuffle(rids, new Random(42)); + for (DataBox key : keys) { + fromDisk.remove(key); + assertEquals(Optional.empty(), fromDisk.get(key)); + } + } + } + } + + @Test + @Category(SystemTests.class) + public void testMaxOrder() { + // Note that this white box test depend critically on the implementation + // of toBytes and includes a lot of magic numbers that won't make sense + // unless you read toBytes. + assertEquals(4, Type.intType().getSizeInBytes()); + assertEquals(8, Type.longType().getSizeInBytes()); + assertEquals(10, RecordId.getSizeInBytes()); + short pageSizeInBytes = 100; + Type keySchema = Type.intType(); + assertEquals(3, LeafNode.maxOrder(pageSizeInBytes, keySchema)); + assertEquals(3, InnerNode.maxOrder(pageSizeInBytes, keySchema)); + assertEquals(3, BPlusTree.maxOrder(pageSizeInBytes, keySchema)); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/index/TestInnerNode.java b/src/test/java/edu/berkeley/cs186/database/index/TestInnerNode.java new file mode 100644 index 0000000..c8b74ac --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/index/TestInnerNode.java @@ -0,0 +1,413 @@ +package edu.berkeley.cs186.database.index; + +import java.util.*; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.MemoryDiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.ClockEvictionPolicy; +import edu.berkeley.cs186.database.recovery.DummyRecoveryManager; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.table.RecordId; + +import static org.junit.Assert.*; + +@Category(HW2Tests.class) +public class TestInnerNode { + private BufferManager bufferManager; + private BPlusTreeMetadata metadata; + private LockContext treeContext; + + // 1 second max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + // inner, leaf0, leaf1, and leaf2 collectively form the following B+ tree: + // + // inner + // +----+----+----+----+ + // | 10 | 20 | | | + // +----+----+----+----+ + // / | \ + // ____/ | \____ + // / | \ + // +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + // | 1 | 2 | 3 | | | 11 | 12 | 13 | | | 21 | 22 | 23 | | + // +----+----+----+----+ +----+----+----+----+ +----+----+----+----+ + // leaf0 leaf1 leaf2 + // + // innerKeys, innerChildren, keys0, rids0, keys1, rids1, keys2, and rids2 + // hold *copies* of the contents of the nodes. To test out a certain method + // of a tree---for example, put---we can issue a put against the tree, + // update one of innerKeys, innerChildren, keys{0,1,2}, or rids{0,1,2}, and + // then check that the contents of the tree match our expectations. For + // example: + // + // IntDataBox key = new IntDataBox(4); + // RecordId rid = new RecordId(4, (short) 4); + // inner.put(key, rid); + // + // // (4, (4, 4)) is added to leaf 0, so we update keys0 and rids0 and + // // check that it matches the contents of leaf0. + // keys0.add(key); + // rids0.add(rid); + // assertEquals(keys0, getLeaf(leaf0).getKeys()); + // assertEquals(rids0, getLeaf(leaf0).getRids()); + // + // // Leaf 1 should be unchanged which we can check: + // assertEquals(keys1, getLeaf(leaf1).getKeys()); + // assertEquals(rids1, getLeaf(leaf1).getRids()); + // + // // Writing all these assertEquals is boilerplate, so we can abstract + // // it in checkTreeMatchesExpectations(). + // checkTreeMatchesExpectations(); + // + // Note that we cannot simply store the LeafNodes as members because when + // we call something like inner.put(k), the inner node constructs a new + // LeafNode from the serialization and forwards the put to that. It would + // not affect our the in-memory values of our members. Also note that all + // of these members are initialized by resetMembers before every test case + // is run. + private List innerKeys; + private List innerChildren; + private InnerNode inner; + private List keys0; + private List rids0; + private long leaf0; + private List keys1; + private List rids1; + private long leaf1; + private List keys2; + private List rids2; + private long leaf2; + + // See comment above. + @Before + public void resetMembers() { + DiskSpaceManager diskSpaceManager = new MemoryDiskSpaceManager(); + diskSpaceManager.allocPart(0); + this.bufferManager = new BufferManagerImpl(diskSpaceManager, new DummyRecoveryManager(), 1024, + new ClockEvictionPolicy()); + this.treeContext = new DummyLockContext(); + setBPlusTreeMetadata(Type.intType(), 2); + + // Leaf 2 + List keys2 = new ArrayList<>(); + keys2.add(new IntDataBox(21)); + keys2.add(new IntDataBox(22)); + keys2.add(new IntDataBox(23)); + List rids2 = new ArrayList<>(); + rids2.add(new RecordId(21, (short) 21)); + rids2.add(new RecordId(22, (short) 22)); + rids2.add(new RecordId(23, (short) 23)); + Optional sibling2 = Optional.empty(); + LeafNode leaf2 = new LeafNode(metadata, bufferManager, keys2, rids2, sibling2, treeContext); + + this.keys2 = new ArrayList<>(keys2); + this.rids2 = new ArrayList<>(rids2); + this.leaf2 = leaf2.getPage().getPageNum(); + + // Leaf 1 + keys1 = new ArrayList<>(); + keys1.add(new IntDataBox(11)); + keys1.add(new IntDataBox(12)); + keys1.add(new IntDataBox(13)); + rids1 = new ArrayList<>(); + rids1.add(new RecordId(11, (short) 11)); + rids1.add(new RecordId(12, (short) 12)); + rids1.add(new RecordId(13, (short) 13)); + Optional sibling1 = Optional.of(leaf2.getPage().getPageNum()); + LeafNode leaf1 = new LeafNode(metadata, bufferManager, keys1, rids1, sibling1, treeContext); + + this.keys1 = new ArrayList<>(keys1); + this.rids1 = new ArrayList<>(rids1); + this.leaf1 = leaf1.getPage().getPageNum(); + + // Leaf 0 + List keys0 = new ArrayList<>(); + keys0.add(new IntDataBox(1)); + keys0.add(new IntDataBox(2)); + keys0.add(new IntDataBox(3)); + List rids0 = new ArrayList<>(); + rids0.add(new RecordId(1, (short) 1)); + rids0.add(new RecordId(2, (short) 2)); + rids0.add(new RecordId(3, (short) 3)); + Optional sibling0 = Optional.of(leaf1.getPage().getPageNum()); + LeafNode leaf0 = new LeafNode(metadata, bufferManager, keys0, rids0, sibling0, treeContext); + this.keys0 = new ArrayList<>(keys0); + this.rids0 = new ArrayList<>(rids0); + this.leaf0 = leaf0.getPage().getPageNum(); + + // Inner node + List innerKeys = new ArrayList<>(); + innerKeys.add(new IntDataBox(10)); + innerKeys.add(new IntDataBox(20)); + + List innerChildren = new ArrayList<>(); + innerChildren.add(this.leaf0); + innerChildren.add(this.leaf1); + innerChildren.add(this.leaf2); + + this.innerKeys = new ArrayList<>(innerKeys); + this.innerChildren = new ArrayList<>(innerChildren); + this.inner = new InnerNode(metadata, bufferManager, innerKeys, innerChildren, treeContext); + } + + @After + public void cleanup() { + this.bufferManager.close(); + } + + private void setBPlusTreeMetadata(Type keySchema, int order) { + this.metadata = new BPlusTreeMetadata("test", "col", keySchema, order, + 0, DiskSpaceManager.INVALID_PAGE_NUM, -1); + } + + // See comment above. + private LeafNode getLeaf(long pageNum) { + return LeafNode.fromBytes(metadata, bufferManager, treeContext, pageNum); + } + + // See comment above. + private void checkTreeMatchesExpectations() { + LeafNode leaf0 = getLeaf(this.leaf0); + LeafNode leaf1 = getLeaf(this.leaf1); + LeafNode leaf2 = getLeaf(this.leaf2); + + assertEquals(keys0, leaf0.getKeys()); + assertEquals(rids0, leaf0.getRids()); + assertEquals(keys1, leaf1.getKeys()); + assertEquals(rids1, leaf1.getRids()); + assertEquals(keys2, leaf2.getKeys()); + assertEquals(rids2, leaf2.getRids()); + assertEquals(innerKeys, inner.getKeys()); + assertEquals(innerChildren, inner.getChildren()); + } + + // Tests /////////////////////////////////////////////////////////////////// + @Test + @Category(PublicTests.class) + public void testGet() { + LeafNode leaf0 = getLeaf(this.leaf0); + assertNotNull(leaf0); + for (int i = 0; i < 10; ++i) { + assertEquals(leaf0, inner.get(new IntDataBox(i))); + } + + LeafNode leaf1 = getLeaf(this.leaf1); + for (int i = 10; i < 20; ++i) { + assertEquals(leaf1, inner.get(new IntDataBox(i))); + } + + LeafNode leaf2 = getLeaf(this.leaf2); + for (int i = 20; i < 30; ++i) { + assertEquals(leaf2, inner.get(new IntDataBox(i))); + } + } + + @Test + @Category(PublicTests.class) + public void testGetLeftmostLeaf() { + assertNotNull(getLeaf(leaf0)); + assertEquals(getLeaf(leaf0), inner.getLeftmostLeaf()); + } + + @Test + @Category(PublicTests.class) + public void testNoOverflowPuts() { + IntDataBox key = null; + RecordId rid = null; + + // Add to leaf 0. + key = new IntDataBox(0); + rid = new RecordId(0, (short) 0); + assertEquals(Optional.empty(), inner.put(key, rid)); + keys0.add(0, key); + rids0.add(0, rid); + checkTreeMatchesExpectations(); + + // Add to leaf 1. + key = new IntDataBox(14); + rid = new RecordId(14, (short) 14); + assertEquals(Optional.empty(), inner.put(key, rid)); + keys1.add(3, key); + rids1.add(3, rid); + checkTreeMatchesExpectations(); + + // Add to leaf 2. + key = new IntDataBox(20); + rid = new RecordId(20, (short) 20); + assertEquals(Optional.empty(), inner.put(key, rid)); + keys2.add(0, key); + rids2.add(0, rid); + checkTreeMatchesExpectations(); + } + + @Test + @Category(PublicTests.class) + public void testRemove() { + // Remove from leaf 0. + inner.remove(new IntDataBox(1)); + keys0.remove(0); + rids0.remove(0); + checkTreeMatchesExpectations(); + + inner.remove(new IntDataBox(3)); + keys0.remove(1); + rids0.remove(1); + checkTreeMatchesExpectations(); + + inner.remove(new IntDataBox(2)); + keys0.remove(0); + rids0.remove(0); + checkTreeMatchesExpectations(); + + // Remove from leaf 1. + inner.remove(new IntDataBox(11)); + keys1.remove(0); + rids1.remove(0); + checkTreeMatchesExpectations(); + + inner.remove(new IntDataBox(13)); + keys1.remove(1); + rids1.remove(1); + checkTreeMatchesExpectations(); + + inner.remove(new IntDataBox(12)); + keys1.remove(0); + rids1.remove(0); + checkTreeMatchesExpectations(); + + // Remove from leaf 2. + inner.remove(new IntDataBox(23)); + keys2.remove(2); + rids2.remove(2); + checkTreeMatchesExpectations(); + + inner.remove(new IntDataBox(22)); + keys2.remove(1); + rids2.remove(1); + checkTreeMatchesExpectations(); + + inner.remove(new IntDataBox(21)); + keys2.remove(0); + rids2.remove(0); + checkTreeMatchesExpectations(); + } + + @Test + @Category(SystemTests.class) + public void testMaxOrder() { + // Note that this white box test depend critically on the implementation + // of toBytes and includes a lot of magic numbers that won't make sense + // unless you read toBytes. + assertEquals(4, Type.intType().getSizeInBytes()); + assertEquals(8, Type.longType().getSizeInBytes()); + for (int d = 0; d < 10; ++d) { + int dd = d + 1; + for (int i = 5 + (2 * d * 4) + ((2 * d + 1) * 8); i < 5 + (2 * dd * 4) + ((2 * dd + 1) * 8); ++i) { + assertEquals(d, InnerNode.maxOrder((short) i, Type.intType())); + } + } + } + + @Test + @Category(SystemTests.class) + public void testnumLessThanEqual() { + List empty = Collections.emptyList(); + assertEquals(0, InnerNode.numLessThanEqual(0, empty)); + + List contiguous = Arrays.asList(1, 2, 3, 4, 5); + assertEquals(0, InnerNode.numLessThanEqual(0, contiguous)); + assertEquals(1, InnerNode.numLessThanEqual(1, contiguous)); + assertEquals(2, InnerNode.numLessThanEqual(2, contiguous)); + assertEquals(3, InnerNode.numLessThanEqual(3, contiguous)); + assertEquals(4, InnerNode.numLessThanEqual(4, contiguous)); + assertEquals(5, InnerNode.numLessThanEqual(5, contiguous)); + assertEquals(5, InnerNode.numLessThanEqual(6, contiguous)); + assertEquals(5, InnerNode.numLessThanEqual(7, contiguous)); + + List sparseWithDuplicates = Arrays.asList(1, 3, 3, 3, 5); + assertEquals(0, InnerNode.numLessThanEqual(0, sparseWithDuplicates)); + assertEquals(1, InnerNode.numLessThanEqual(1, sparseWithDuplicates)); + assertEquals(1, InnerNode.numLessThanEqual(2, sparseWithDuplicates)); + assertEquals(4, InnerNode.numLessThanEqual(3, sparseWithDuplicates)); + assertEquals(4, InnerNode.numLessThanEqual(4, sparseWithDuplicates)); + assertEquals(5, InnerNode.numLessThanEqual(5, sparseWithDuplicates)); + assertEquals(5, InnerNode.numLessThanEqual(6, sparseWithDuplicates)); + assertEquals(5, InnerNode.numLessThanEqual(7, sparseWithDuplicates)); + } + + @Test + @Category(SystemTests.class) + public void testnumLessThan() { + List empty = Collections.emptyList(); + assertEquals(0, InnerNode.numLessThanEqual(0, empty)); + + List contiguous = Arrays.asList(1, 2, 3, 4, 5); + assertEquals(0, InnerNode.numLessThan(0, contiguous)); + assertEquals(0, InnerNode.numLessThan(1, contiguous)); + assertEquals(1, InnerNode.numLessThan(2, contiguous)); + assertEquals(2, InnerNode.numLessThan(3, contiguous)); + assertEquals(3, InnerNode.numLessThan(4, contiguous)); + assertEquals(4, InnerNode.numLessThan(5, contiguous)); + assertEquals(5, InnerNode.numLessThan(6, contiguous)); + assertEquals(5, InnerNode.numLessThan(7, contiguous)); + + List sparseWithDuplicates = Arrays.asList(1, 3, 3, 3, 5); + assertEquals(0, InnerNode.numLessThan(0, sparseWithDuplicates)); + assertEquals(0, InnerNode.numLessThan(1, sparseWithDuplicates)); + assertEquals(1, InnerNode.numLessThan(2, sparseWithDuplicates)); + assertEquals(1, InnerNode.numLessThan(3, sparseWithDuplicates)); + assertEquals(4, InnerNode.numLessThan(4, sparseWithDuplicates)); + assertEquals(4, InnerNode.numLessThan(5, sparseWithDuplicates)); + assertEquals(5, InnerNode.numLessThan(6, sparseWithDuplicates)); + assertEquals(5, InnerNode.numLessThan(7, sparseWithDuplicates)); + } + + @Test + @Category(PublicTests.class) + public void testToSexp() { + String leaf0 = "((1 (1 1)) (2 (2 2)) (3 (3 3)))"; + String leaf1 = "((11 (11 11)) (12 (12 12)) (13 (13 13)))"; + String leaf2 = "((21 (21 21)) (22 (22 22)) (23 (23 23)))"; + String expected = String.format("(%s 10 %s 20 %s)", leaf0, leaf1, leaf2); + assertEquals(expected, inner.toSexp()); + } + + @Test + @Category(SystemTests.class) + public void testToAndFromBytes() { + int d = 5; + setBPlusTreeMetadata(Type.intType(), d); + + List keys = new ArrayList<>(); + List children = new ArrayList<>(); + children.add(42L); + + for (int i = 0; i < 2 * d; ++i) { + keys.add(new IntDataBox(i)); + children.add((long) i); + + InnerNode inner = new InnerNode(metadata, bufferManager, keys, children, treeContext); + long pageNum = inner.getPage().getPageNum(); + InnerNode parsed = InnerNode.fromBytes(metadata, bufferManager, treeContext, pageNum); + assertEquals(inner, parsed); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/index/TestLeafNode.java b/src/test/java/edu/berkeley/cs186/database/index/TestLeafNode.java new file mode 100644 index 0000000..fc63d67 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/index/TestLeafNode.java @@ -0,0 +1,309 @@ +package edu.berkeley.cs186.database.index; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Random; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.MemoryDiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.ClockEvictionPolicy; +import edu.berkeley.cs186.database.recovery.DummyRecoveryManager; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.Type; +import edu.berkeley.cs186.database.table.RecordId; + +@Category(HW2Tests.class) +public class TestLeafNode { + private BufferManager bufferManager; + private BPlusTreeMetadata metadata; + private LockContext treeContext; + + // 1 second max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + private static DataBox d0 = new IntDataBox(0); + private static DataBox d1 = new IntDataBox(1); + private static DataBox d2 = new IntDataBox(2); + private static DataBox d3 = new IntDataBox(3); + private static DataBox d4 = new IntDataBox(4); + + private static RecordId r0 = new RecordId(0, (short) 0); + private static RecordId r1 = new RecordId(1, (short) 1); + private static RecordId r2 = new RecordId(2, (short) 2); + private static RecordId r3 = new RecordId(3, (short) 3); + private static RecordId r4 = new RecordId(4, (short) 4); + + @Before + public void setup() { + DiskSpaceManager diskSpaceManager = new MemoryDiskSpaceManager(); + diskSpaceManager.allocPart(0); + this.bufferManager = new BufferManagerImpl(diskSpaceManager, new DummyRecoveryManager(), 1024, + new ClockEvictionPolicy()); + this.treeContext = new DummyLockContext(); + this.metadata = null; + } + + @After + public void cleanup() { + this.bufferManager.close(); + } + + // Helpers ///////////////////////////////////////////////////////////////// + private void setBPlusTreeMetadata(Type keySchema, int order) { + this.metadata = new BPlusTreeMetadata("test", "col", keySchema, order, + 0, DiskSpaceManager.INVALID_PAGE_NUM, -1); + } + + private LeafNode getEmptyLeaf(Optional rightSibling) { + List keys = new ArrayList<>(); + List rids = new ArrayList<>(); + return new LeafNode(metadata, bufferManager, keys, rids, rightSibling, treeContext); + } + + // Tests /////////////////////////////////////////////////////////////////// + @Test + @Category(PublicTests.class) + public void testGetL() { + setBPlusTreeMetadata(Type.intType(), 5); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + for (int i = 0; i < 10; ++i) { + assertEquals(leaf, leaf.get(new IntDataBox(i))); + } + } + + @Test + @Category(PublicTests.class) + public void testGetLeftmostLeafL() { + setBPlusTreeMetadata(Type.intType(), 5); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + assertEquals(leaf, leaf.getLeftmostLeaf()); + } + + @Test + @Category(PublicTests.class) + public void testSmallBulkLoad() { + // Bulk loads with 60% of a leaf's worth, then checks that the + // leaf didn't split. + int d = 5; + float fillFactor = 0.8f; + setBPlusTreeMetadata(Type.intType(), d); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + + List> data = new ArrayList<>(); + for (int i = 0; i < (int) Math.ceil(1.5 * d * fillFactor); ++i) { + DataBox key = new IntDataBox(i); + RecordId rid = new RecordId(i, (short) i); + data.add(i, new Pair<>(key, rid)); + } + + assertFalse(leaf.bulkLoad(data.iterator(), fillFactor).isPresent()); + + Iterator iter = leaf.scanAll(); + Iterator> expected = data.iterator(); + while (iter.hasNext() && expected.hasNext()) { + assertEquals(expected.next().getSecond(), iter.next()); + } + assertFalse(iter.hasNext()); + assertFalse(expected.hasNext()); + } + + @Test + @Category(PublicTests.class) + public void testNoOverflowPuts() { + int d = 5; + setBPlusTreeMetadata(Type.intType(), d); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + + for (int i = 0; i < 2 * d; ++i) { + DataBox key = new IntDataBox(i); + RecordId rid = new RecordId(i, (short) i); + assertEquals(Optional.empty(), leaf.put(key, rid)); + + for (int j = 0; j <= i; ++j) { + key = new IntDataBox(j); + rid = new RecordId(j, (short) j); + assertEquals(Optional.of(rid), leaf.getKey(key)); + } + } + } + + @Test + @Category(PublicTests.class) + public void testNoOverflowPutsFromDisk() { + int d = 5; + setBPlusTreeMetadata(Type.intType(), d); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + + // Populate the leaf. + for (int i = 0; i < 2 * d; ++i) { + leaf.put(new IntDataBox(i), new RecordId(i, (short) i)); + } + + // Then read the leaf from disk. + long pageNum = leaf.getPage().getPageNum(); + LeafNode fromDisk = LeafNode.fromBytes(metadata, bufferManager, treeContext, pageNum); + + // Check to see that we can read from disk. + for (int i = 0; i < 2 * d; ++i) { + IntDataBox key = new IntDataBox(i); + RecordId rid = new RecordId(i, (short) i); + assertEquals(Optional.of(rid), fromDisk.getKey(key)); + } + } + + @Test(expected = BPlusTreeException.class) + @Category(PublicTests.class) + public void testDuplicatePut() { + setBPlusTreeMetadata(Type.intType(), 4); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + + // The initial insert is fine. + leaf.put(new IntDataBox(0), new RecordId(0, (short) 0)); + + // The duplicate insert should raise an exception. + leaf.put(new IntDataBox(0), new RecordId(0, (short) 0)); + } + + @Test + @Category(PublicTests.class) + public void testSimpleRemoves() { + int d = 5; + setBPlusTreeMetadata(Type.intType(), d); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + + // Insert entries. + for (int i = 0; i < 2 * d; ++i) { + IntDataBox key = new IntDataBox(i); + RecordId rid = new RecordId(i, (short) i); + leaf.put(key, rid); + assertEquals(Optional.of(rid), leaf.getKey(key)); + } + + // Remove entries. + for (int i = 0; i < 2 * d; ++i) { + IntDataBox key = new IntDataBox(i); + leaf.remove(key); + assertEquals(Optional.empty(), leaf.getKey(key)); + } + } + + @Test + @Category(PublicTests.class) + public void testScanAll() { + int d = 5; + setBPlusTreeMetadata(Type.intType(), d); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + + // Insert tuples in reverse order to make sure that scanAll is returning + // things in sorted order. + for (int i = 2 * d - 1; i >= 0; --i) { + leaf.put(new IntDataBox(i), new RecordId(i, (short) i)); + } + + Iterator iter = leaf.scanAll(); + for (int i = 0; i < 2 * d; ++i) { + assertTrue(iter.hasNext()); + assertEquals(new RecordId(i, (short) i), iter.next()); + } + assertFalse(iter.hasNext()); + } + + @Test + @Category(PublicTests.class) + public void testScanGreaterEqual() { + int d = 5; + setBPlusTreeMetadata(Type.intType(), d); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + + // Insert tuples in reverse order to make sure that scanAll is returning + // things in sorted order. + for (int i = 2 * d - 1; i >= 0; --i) { + leaf.put(new IntDataBox(i), new RecordId(i, (short) i)); + } + + Iterator iter = leaf.scanGreaterEqual(new IntDataBox(5)); + for (int i = 5; i < 2 * d; ++i) { + assertTrue(iter.hasNext()); + assertEquals(new RecordId(i, (short) i), iter.next()); + } + assertFalse(iter.hasNext()); + } + + @Test + @Category(SystemTests.class) + public void testMaxOrder() { + // Note that this white box test depend critically on the implementation + // of toBytes and includes a lot of magic numbers that won't make sense + // unless you read toBytes. + assertEquals(4, Type.intType().getSizeInBytes()); + assertEquals(8, Type.longType().getSizeInBytes()); + assertEquals(10, RecordId.getSizeInBytes()); + for (int d = 0; d < 10; ++d) { + int dd = d + 1; + for (int i = 13 + (2 * d) * (4 + 10); i < 13 + (2 * dd) * (4 + 10); ++i) { + assertEquals(d, LeafNode.maxOrder((short) i, Type.intType())); + } + } + } + + @Test + @Category(PublicTests.class) + public void testToSexp() { + int d = 2; + setBPlusTreeMetadata(Type.intType(), d); + LeafNode leaf = getEmptyLeaf(Optional.empty()); + + assertEquals("()", leaf.toSexp()); + leaf.put(new IntDataBox(4), new RecordId(4, (short) 4)); + assertEquals("((4 (4 4)))", leaf.toSexp()); + leaf.put(new IntDataBox(1), new RecordId(1, (short) 1)); + assertEquals("((1 (1 1)) (4 (4 4)))", leaf.toSexp()); + leaf.put(new IntDataBox(2), new RecordId(2, (short) 2)); + assertEquals("((1 (1 1)) (2 (2 2)) (4 (4 4)))", leaf.toSexp()); + leaf.put(new IntDataBox(3), new RecordId(3, (short) 3)); + assertEquals("((1 (1 1)) (2 (2 2)) (3 (3 3)) (4 (4 4)))", leaf.toSexp()); + } + + @Test + @Category(PublicTests.class) + public void testToAndFromBytes() { + int d = 5; + setBPlusTreeMetadata(Type.intType(), d); + + List keys = new ArrayList<>(); + List rids = new ArrayList<>(); + LeafNode leaf = new LeafNode(metadata, bufferManager, keys, rids, Optional.of(42L), treeContext); + long pageNum = leaf.getPage().getPageNum(); + + assertEquals(leaf, LeafNode.fromBytes(metadata, bufferManager, treeContext, pageNum)); + + for (int i = 0; i < 2 * d; ++i) { + leaf.put(new IntDataBox(i), new RecordId(i, (short) i)); + assertEquals(leaf, LeafNode.fromBytes(metadata, bufferManager, treeContext, pageNum)); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/io/MemoryDiskSpaceManager.java b/src/test/java/edu/berkeley/cs186/database/io/MemoryDiskSpaceManager.java new file mode 100644 index 0000000..f8d40c1 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/io/MemoryDiskSpaceManager.java @@ -0,0 +1,112 @@ +package edu.berkeley.cs186.database.io; + +import java.util.*; + +/** + * "Disk" space manager that really just keeps things in memory. Not thread safe. + */ +public class MemoryDiskSpaceManager implements DiskSpaceManager { + private Map> partitions = new HashMap<>(); + private Map nextPageNum = new HashMap<>(); + private Map pages = new HashMap<>(); + private int nextPartitionNum = 0; + + @Override + public void close() {} + + @Override + public int allocPart() { + partitions.put(nextPartitionNum, new HashSet<>()); + nextPageNum.put(nextPartitionNum, 0); + return nextPartitionNum++; + } + + @Override + public int allocPart(int partNum) { + if (partitions.containsKey(partNum)) { + throw new IllegalStateException("partition " + partNum + " already allocated"); + } + partitions.put(partNum, new HashSet<>()); + nextPageNum.put(partNum, 0); + nextPartitionNum = partNum + 1; + return partNum; + } + + @Override + public void freePart(int partNum) { + if (!partitions.containsKey(partNum)) { + throw new NoSuchElementException("partition " + partNum + " not allocated"); + } + for (Integer pageNum : partitions.remove(partNum)) { + pages.remove(DiskSpaceManager.getVirtualPageNum(partNum, pageNum)); + } + nextPageNum.remove(partNum); + } + + @Override + public long allocPage(int partNum) { + if (!partitions.containsKey(partNum)) { + throw new IllegalArgumentException("partition " + partNum + " not allocated"); + } + int ppageNum = nextPageNum.get(partNum); + nextPageNum.put(partNum, ppageNum + 1); + long pageNum = DiskSpaceManager.getVirtualPageNum(partNum, ppageNum); + partitions.get(partNum).add(ppageNum); + pages.put(pageNum, new byte[DiskSpaceManager.PAGE_SIZE]); + return pageNum; + } + + @Override + public long allocPage(long page) { + int partNum = DiskSpaceManager.getPartNum(page); + int ppageNum = DiskSpaceManager.getPageNum(page); + if (!partitions.containsKey(partNum)) { + throw new IllegalArgumentException("partition " + partNum + " not allocated"); + } + if (pages.containsKey(page)) { + throw new IllegalStateException("page " + page + " already allocated"); + } + nextPageNum.put(partNum, ppageNum + 1); + partitions.get(partNum).add(ppageNum); + pages.put(page, new byte[DiskSpaceManager.PAGE_SIZE]); + return page; + } + + @Override + public void freePage(long page) { + if (!pages.containsKey(page)) { + throw new NoSuchElementException("page " + page + " not allocated"); + } + int partNum = DiskSpaceManager.getPartNum(page); + int pageNum = DiskSpaceManager.getPageNum(page); + partitions.get(partNum).remove(pageNum); + pages.remove(page); + } + + @Override + public void readPage(long page, byte[] buf) { + if (buf.length != DiskSpaceManager.PAGE_SIZE) { + throw new IllegalArgumentException("bad buffer size"); + } + if (!pages.containsKey(page)) { + throw new PageException("page " + page + " not allocated"); + } + System.arraycopy(pages.get(page), 0, buf, 0, DiskSpaceManager.PAGE_SIZE); + } + + @Override + public void writePage(long page, byte[] buf) { + if (buf.length != DiskSpaceManager.PAGE_SIZE) { + throw new IllegalArgumentException("bad buffer size"); + } + if (!pages.containsKey(page)) { + throw new PageException("page " + page + " not allocated"); + } + System.arraycopy(buf, 0, pages.get(page), 0, DiskSpaceManager.PAGE_SIZE); + } + + @Override + public boolean pageAllocated(long page) { + return pages.containsKey(page); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/io/TestDiskSpaceManager.java b/src/test/java/edu/berkeley/cs186/database/io/TestDiskSpaceManager.java new file mode 100644 index 0000000..13e303d --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/io/TestDiskSpaceManager.java @@ -0,0 +1,302 @@ +package edu.berkeley.cs186.database.io; + +import edu.berkeley.cs186.database.categories.HW99Tests; +import edu.berkeley.cs186.database.categories.SystemTests; +import edu.berkeley.cs186.database.recovery.DummyRecoveryManager; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.NoSuchElementException; + +import static org.junit.Assert.*; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestDiskSpaceManager { + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private DiskSpaceManager diskSpaceManager; + private Path managerRoot; + + @Before + public void beforeEach() throws IOException { + managerRoot = tempFolder.newFolder("dsm-test").toPath(); + } + + private DiskSpaceManager getDiskSpaceManager() { + return new DiskSpaceManagerImpl(managerRoot.toString(), new DummyRecoveryManager()); + } + + @Test + public void testCreateDiskSpaceManager() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.close(); + } + + @Test + public void testAllocPart() { + diskSpaceManager = getDiskSpaceManager(); + int partNum = diskSpaceManager.allocPart(0); + + assertEquals(0, partNum); + assertTrue(managerRoot.resolve("0").toFile().exists()); + assertEquals(DiskSpaceManager.PAGE_SIZE, managerRoot.resolve("0").toFile().length()); + + partNum = diskSpaceManager.allocPart(); + + assertEquals(1, partNum); + assertTrue(managerRoot.resolve("1").toFile().exists()); + assertEquals(DiskSpaceManager.PAGE_SIZE, managerRoot.resolve("1").toFile().length()); + + diskSpaceManager.close(); + } + + @Test + public void testAllocPartPersist() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.allocPart(); + diskSpaceManager.close(); + diskSpaceManager = getDiskSpaceManager(); + int partNum = diskSpaceManager.allocPart(); + diskSpaceManager.close(); + + assertEquals(1, partNum); + assertTrue(managerRoot.resolve("1").toFile().exists()); + assertEquals(DiskSpaceManager.PAGE_SIZE, managerRoot.resolve("0").toFile().length()); + } + + @Test(expected = NoSuchElementException.class) + public void testFreePartBad() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.freePart(1); + diskSpaceManager.close(); + } + + @Test + public void testFreePart() { + diskSpaceManager = getDiskSpaceManager(); + int partNum = diskSpaceManager.allocPart(); + diskSpaceManager.freePart(partNum); + + assertFalse(managerRoot.resolve("0").toFile().exists()); + + diskSpaceManager.close(); + } + + @Test + public void testFreePartPersist() { + diskSpaceManager = getDiskSpaceManager(); + int partNum = diskSpaceManager.allocPart(); + diskSpaceManager.freePart(partNum); + diskSpaceManager.close(); + diskSpaceManager = getDiskSpaceManager(); + partNum = diskSpaceManager.allocPart(); + diskSpaceManager.close(); + + assertEquals(0, partNum); + assertTrue(managerRoot.resolve("0").toFile().exists()); + assertFalse(managerRoot.resolve("1").toFile().exists()); + } + + @Test + public void testAllocPageZeroed() { + diskSpaceManager = getDiskSpaceManager(); + int partNum = diskSpaceManager.allocPart(0); + long pageNum1 = diskSpaceManager.allocPage(0); + long pageNum2 = diskSpaceManager.allocPage(partNum); + + assertEquals(0L, pageNum1); + assertEquals(1L, pageNum2); + + byte[] buf = new byte[DiskSpaceManager.PAGE_SIZE]; + diskSpaceManager.readPage(pageNum1, buf); + assertArrayEquals(new byte[DiskSpaceManager.PAGE_SIZE], buf); + diskSpaceManager.readPage(pageNum2, buf); + assertArrayEquals(new byte[DiskSpaceManager.PAGE_SIZE], buf); + + long pageNum3 = diskSpaceManager.allocPage(partNum); + long pageNum4 = diskSpaceManager.allocPage(partNum); + diskSpaceManager.close(); + + diskSpaceManager = getDiskSpaceManager(); + + diskSpaceManager.readPage(pageNum1, buf); + assertArrayEquals(new byte[DiskSpaceManager.PAGE_SIZE], buf); + diskSpaceManager.readPage(pageNum2, buf); + assertArrayEquals(new byte[DiskSpaceManager.PAGE_SIZE], buf); + diskSpaceManager.readPage(pageNum3, buf); + assertArrayEquals(new byte[DiskSpaceManager.PAGE_SIZE], buf); + diskSpaceManager.readPage(pageNum4, buf); + assertArrayEquals(new byte[DiskSpaceManager.PAGE_SIZE], buf); + + diskSpaceManager.close(); + } + + @Test(expected = NoSuchElementException.class) + public void testReadBadPart() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.readPage(0, new byte[DiskSpaceManager.PAGE_SIZE]); + diskSpaceManager.close(); + } + + @Test(expected = NoSuchElementException.class) + public void testWriteBadPart() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.writePage(0, new byte[DiskSpaceManager.PAGE_SIZE]); + diskSpaceManager.close(); + } + + @Test(expected = IllegalArgumentException.class) + public void testReadBadBuffer() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.readPage(0, new byte[DiskSpaceManager.PAGE_SIZE - 1]); + diskSpaceManager.close(); + } + + @Test(expected = IllegalArgumentException.class) + public void testWriteBadBuffer() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.writePage(0, new byte[DiskSpaceManager.PAGE_SIZE + 1]); + diskSpaceManager.close(); + } + + @Test(expected = PageException.class) + public void testReadOutOfBounds() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.allocPart(); + diskSpaceManager.readPage(0, new byte[DiskSpaceManager.PAGE_SIZE]); + diskSpaceManager.close(); + } + + @Test(expected = PageException.class) + public void testWriteOutOfBounds() { + diskSpaceManager = getDiskSpaceManager(); + diskSpaceManager.allocPart(); + diskSpaceManager.writePage(0, new byte[DiskSpaceManager.PAGE_SIZE]); + diskSpaceManager.close(); + } + + @Test + public void testReadWrite() { + diskSpaceManager = getDiskSpaceManager(); + int partNum = diskSpaceManager.allocPart(); + long pageNum = diskSpaceManager.allocPage(partNum); + + byte[] buf = new byte[DiskSpaceManager.PAGE_SIZE]; + for (int i = 0; i < buf.length; ++i) { + buf[i] = (byte) (Integer.valueOf(i).hashCode() & 0xFF); + } + diskSpaceManager.writePage(pageNum, buf); + byte[] readbuf = new byte[DiskSpaceManager.PAGE_SIZE]; + diskSpaceManager.readPage(pageNum, readbuf); + + assertArrayEquals(buf, readbuf); + + diskSpaceManager.freePart(partNum); + diskSpaceManager.close(); + } + + @Test + public void testReadWritePersistent() { + diskSpaceManager = getDiskSpaceManager(); + int partNum = diskSpaceManager.allocPart(); + long pageNum = diskSpaceManager.allocPage(partNum); + + byte[] buf = new byte[DiskSpaceManager.PAGE_SIZE]; + for (int i = 0; i < buf.length; ++i) { + buf[i] = (byte) (Integer.valueOf(i).hashCode() & 0xFF); + } + diskSpaceManager.writePage(pageNum, buf); + diskSpaceManager.close(); + + diskSpaceManager = getDiskSpaceManager(); + byte[] readbuf = new byte[DiskSpaceManager.PAGE_SIZE]; + diskSpaceManager.readPage(pageNum, readbuf); + + assertArrayEquals(buf, readbuf); + + diskSpaceManager.freePart(partNum); + diskSpaceManager.close(); + } + + @Test + public void testReadWriteMultiplePartitions() { + diskSpaceManager = getDiskSpaceManager(); + int partNum1 = diskSpaceManager.allocPart(); + int partNum2 = diskSpaceManager.allocPart(); + long pageNum11 = diskSpaceManager.allocPage(partNum1); + long pageNum21 = diskSpaceManager.allocPage(partNum2); + long pageNum22 = diskSpaceManager.allocPage(partNum2); + + byte[] buf1 = new byte[DiskSpaceManager.PAGE_SIZE]; + byte[] buf2 = new byte[DiskSpaceManager.PAGE_SIZE]; + byte[] buf3 = new byte[DiskSpaceManager.PAGE_SIZE]; + for (int i = 0; i < buf1.length; ++i) { + buf1[i] = (byte) (Integer.valueOf(i).hashCode() & 0xFF); + buf2[i] = (byte) ((Integer.valueOf(i).hashCode() >> 8) & 0xFF); + buf3[i] = (byte) ((Integer.valueOf(i).hashCode() >> 16) & 0xFF); + } + diskSpaceManager.writePage(pageNum11, buf1); + diskSpaceManager.writePage(pageNum22, buf3); + diskSpaceManager.writePage(pageNum21, buf2); + byte[] readbuf1 = new byte[DiskSpaceManager.PAGE_SIZE]; + byte[] readbuf2 = new byte[DiskSpaceManager.PAGE_SIZE]; + byte[] readbuf3 = new byte[DiskSpaceManager.PAGE_SIZE]; + diskSpaceManager.readPage(pageNum11, readbuf1); + diskSpaceManager.readPage(pageNum21, readbuf2); + diskSpaceManager.readPage(pageNum22, readbuf3); + + assertArrayEquals(buf1, readbuf1); + assertArrayEquals(buf2, readbuf2); + assertArrayEquals(buf3, readbuf3); + + diskSpaceManager.freePart(partNum1); + diskSpaceManager.freePart(partNum2); + diskSpaceManager.close(); + } + + @Test + public void testReadWriteMultiplePartitionsPersistent() { + diskSpaceManager = getDiskSpaceManager(); + int partNum1 = diskSpaceManager.allocPart(); + int partNum2 = diskSpaceManager.allocPart(); + long pageNum11 = diskSpaceManager.allocPage(partNum1); + long pageNum21 = diskSpaceManager.allocPage(partNum2); + long pageNum22 = diskSpaceManager.allocPage(partNum2); + + byte[] buf1 = new byte[DiskSpaceManager.PAGE_SIZE]; + byte[] buf2 = new byte[DiskSpaceManager.PAGE_SIZE]; + byte[] buf3 = new byte[DiskSpaceManager.PAGE_SIZE]; + for (int i = 0; i < buf1.length; ++i) { + buf1[i] = (byte) (Integer.valueOf(i).hashCode() & 0xFF); + buf2[i] = (byte) ((Integer.valueOf(i).hashCode() >> 8) & 0xFF); + buf3[i] = (byte) ((Integer.valueOf(i).hashCode() >> 16) & 0xFF); + } + diskSpaceManager.writePage(pageNum11, buf1); + diskSpaceManager.writePage(pageNum22, buf3); + diskSpaceManager.writePage(pageNum21, buf2); + diskSpaceManager.close(); + + diskSpaceManager = getDiskSpaceManager(); + byte[] readbuf1 = new byte[DiskSpaceManager.PAGE_SIZE]; + byte[] readbuf2 = new byte[DiskSpaceManager.PAGE_SIZE]; + byte[] readbuf3 = new byte[DiskSpaceManager.PAGE_SIZE]; + diskSpaceManager.readPage(pageNum11, readbuf1); + diskSpaceManager.readPage(pageNum21, readbuf2); + diskSpaceManager.readPage(pageNum22, readbuf3); + + assertArrayEquals(buf1, readbuf1); + assertArrayEquals(buf2, readbuf2); + assertArrayEquals(buf3, readbuf3); + + diskSpaceManager.freePart(partNum1); + diskSpaceManager.freePart(partNum2); + diskSpaceManager.close(); + } +} + diff --git a/src/test/java/edu/berkeley/cs186/database/memory/TestBufferManager.java b/src/test/java/edu/berkeley/cs186/database/memory/TestBufferManager.java new file mode 100644 index 0000000..426ebcb --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/memory/TestBufferManager.java @@ -0,0 +1,333 @@ +package edu.berkeley.cs186.database.memory; + +import edu.berkeley.cs186.database.categories.HW99Tests; +import edu.berkeley.cs186.database.categories.SystemTests; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.MemoryDiskSpaceManager; +import edu.berkeley.cs186.database.io.PageException; +import edu.berkeley.cs186.database.recovery.DummyRecoveryManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.Arrays; + +import static org.junit.Assert.*; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestBufferManager { + private DiskSpaceManager diskSpaceManager; + private BufferManager bufferManager; + + @Before + public void beforeEach() { + diskSpaceManager = new MemoryDiskSpaceManager(); + bufferManager = new BufferManagerImpl(diskSpaceManager, new DummyRecoveryManager(), 5, + new ClockEvictionPolicy()); + } + + @After + public void afterEach() { + bufferManager.close(); + diskSpaceManager.close(); + } + + @Test + public void testFetchNewPage() { + int partNum = diskSpaceManager.allocPart(); + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum, false); + BufferFrame frame2 = bufferManager.fetchNewPageFrame(partNum, false); + BufferFrame frame3 = bufferManager.fetchNewPageFrame(partNum, false); + frame1.unpin(); + frame2.unpin(); + frame3.unpin(); + + assertTrue(frame1.isValid()); + assertTrue(frame2.isValid()); + assertTrue(frame3.isValid()); + + BufferFrame frame4 = bufferManager.fetchNewPageFrame(partNum, false); + BufferFrame frame5 = bufferManager.fetchNewPageFrame(partNum, false); + BufferFrame frame6 = bufferManager.fetchNewPageFrame(partNum, false); + frame4.unpin(); + frame5.unpin(); + frame6.unpin(); + + assertFalse(frame1.isValid()); + assertTrue(frame2.isValid()); + assertTrue(frame3.isValid()); + assertTrue(frame4.isValid()); + assertTrue(frame5.isValid()); + assertTrue(frame6.isValid()); + } + + @Test + public void testFetchPage() { + int partNum = diskSpaceManager.allocPart(); + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum, false); + BufferFrame frame2 = bufferManager.fetchPageFrame(frame1.getPageNum(), false); + + frame1.unpin(); + frame2.unpin(); + + assertSame(frame1, frame2); + } + + @Test + public void testReadWrite() { + int partNum = diskSpaceManager.allocPart(); + + byte[] expected = new byte[] { (byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF }; + byte[] actual = new byte[4]; + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum, false); + frame1.writeBytes((short) 67, (short) 4, expected); + frame1.readBytes((short) 67, (short) 4, actual); + frame1.unpin(); + + assertArrayEquals(expected, actual); + } + + @Test + public void testFlush() { + int partNum = diskSpaceManager.allocPart(); + + byte[] expected = new byte[] { (byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF }; + byte[] actual = new byte[DiskSpaceManager.PAGE_SIZE]; + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum, false); + frame1.writeBytes((short) 67, (short) 4, expected); + frame1.unpin(); + + diskSpaceManager.readPage(frame1.getPageNum(), actual); + assertArrayEquals(new byte[4], Arrays.copyOfRange(actual, 67 + BufferManager.RESERVED_SPACE, + 71 + BufferManager.RESERVED_SPACE)); + + frame1.flush(); + diskSpaceManager.readPage(frame1.getPageNum(), actual); + assertArrayEquals(expected, Arrays.copyOfRange(actual, 67 + BufferManager.RESERVED_SPACE, + 71 + BufferManager.RESERVED_SPACE)); + + frame1.pin(); + frame1.writeBytes((short) 33, (short) 4, expected); + frame1.unpin(); + + diskSpaceManager.readPage(frame1.getPageNum(), actual); + assertArrayEquals(new byte[4], Arrays.copyOfRange(actual, 33 + BufferManager.RESERVED_SPACE, + 37 + BufferManager.RESERVED_SPACE)); + + // force a eviction + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + + diskSpaceManager.readPage(frame1.getPageNum(), actual); + assertFalse(frame1.isValid()); + assertArrayEquals(expected, Arrays.copyOfRange(actual, 33 + BufferManager.RESERVED_SPACE, + 37 + BufferManager.RESERVED_SPACE)); + } + + @Test + public void testFlushLogPage() { + int partNum = diskSpaceManager.allocPart(); + + byte[] expected = new byte[] { (byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF }; + byte[] actual = new byte[DiskSpaceManager.PAGE_SIZE]; + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum, true); + frame1.writeBytes((short) 67, (short) 4, expected); + frame1.unpin(); + + diskSpaceManager.readPage(frame1.getPageNum(), actual); + assertArrayEquals(new byte[4], Arrays.copyOfRange(actual, 67, 71)); + + frame1.flush(); + diskSpaceManager.readPage(frame1.getPageNum(), actual); + assertArrayEquals(expected, Arrays.copyOfRange(actual, 67, 71)); + + frame1.pin(); + frame1.writeBytes((short) 33, (short) 4, expected); + frame1.unpin(); + + diskSpaceManager.readPage(frame1.getPageNum(), actual); + assertArrayEquals(new byte[4], Arrays.copyOfRange(actual, 33, 37)); + + // force a eviction + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + + diskSpaceManager.readPage(frame1.getPageNum(), actual); + assertFalse(frame1.isValid()); + assertArrayEquals(expected, Arrays.copyOfRange(actual, 33, 37)); + } + + @Test + public void testReload() { + int partNum = diskSpaceManager.allocPart(); + + byte[] expected = new byte[] { (byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF }; + byte[] actual = new byte[4]; + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum, false); + frame1.writeBytes((short) 67, (short) 4, expected); + frame1.unpin(); + + // force a eviction + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + + assertFalse(frame1.isValid()); + + // reload page + frame1 = bufferManager.fetchPageFrame(frame1.getPageNum(), false); + frame1.readBytes((short) 67, (short) 4, actual); + frame1.unpin(); + + assertArrayEquals(expected, actual); + } + + @Test + public void testRequestValidFrame() { + int partNum = diskSpaceManager.allocPart(); + + byte[] expected = new byte[] { (byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF }; + byte[] actual = new byte[4]; + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum, false); + frame1.writeBytes((short) 67, (short) 4, expected); + frame1.unpin(); + + assertSame(frame1, frame1.requestValidFrame()); + frame1.unpin(); + + // force a eviction + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + bufferManager.fetchNewPageFrame(partNum, false).unpin(); + + assertFalse(frame1.isValid()); + + BufferFrame frame2 = frame1.requestValidFrame(); + assertNotSame(frame1, frame2); + frame2.readBytes((short) 67, (short) 4, actual); + frame2.unpin(); + + assertArrayEquals(expected, actual); + } + + @Test + public void testFreePage() { + int partNum = diskSpaceManager.allocPart(); + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum, false); + BufferFrame frame2 = bufferManager.fetchNewPageFrame(partNum, false); + Page page3 = bufferManager.fetchNewPage(new DummyLockContext(), partNum, false); + BufferFrame frame4 = bufferManager.fetchNewPageFrame(partNum, false); + BufferFrame frame5 = bufferManager.fetchNewPageFrame(partNum, false); + + frame1.unpin(); + frame2.unpin(); + frame4.unpin(); + frame5.unpin(); + + bufferManager.freePage(page3); + try { + diskSpaceManager.readPage(page3.getPageNum(), new byte[DiskSpaceManager.PAGE_SIZE]); + fail(); + } catch (PageException e) { /* do nothing */ } + + BufferFrame frame6 = bufferManager.fetchNewPageFrame(partNum, false); + frame6.unpin(); + assertTrue(frame1.isValid()); + assertTrue(frame2.isValid()); + assertTrue(frame4.isValid()); + assertTrue(frame5.isValid()); + assertTrue(frame6.isValid()); + } + + @Test + public void testFreePart() { + int partNum1 = diskSpaceManager.allocPart(); + int partNum2 = diskSpaceManager.allocPart(); + + BufferFrame frame1 = bufferManager.fetchNewPageFrame(partNum1, false); + BufferFrame frame2 = bufferManager.fetchNewPageFrame(partNum2, false); + BufferFrame frame3 = bufferManager.fetchNewPageFrame(partNum1, false); + BufferFrame frame4 = bufferManager.fetchNewPageFrame(partNum2, false); + BufferFrame frame5 = bufferManager.fetchNewPageFrame(partNum2, false); + + frame1.unpin(); + frame2.unpin(); + frame3.unpin(); + frame4.unpin(); + frame5.unpin(); + + bufferManager.freePart(partNum1); + + try { + diskSpaceManager.readPage(frame1.getPageNum(), new byte[DiskSpaceManager.PAGE_SIZE]); + fail(); + } catch (Exception e) { /* do nothing */ } + try { + diskSpaceManager.readPage(frame3.getPageNum(), new byte[DiskSpaceManager.PAGE_SIZE]); + fail(); + } catch (Exception e) { /* do nothing */ } + try { + diskSpaceManager.allocPage(partNum1); + fail(); + } catch (Exception e) { /* do nothing */ } + + BufferFrame frame6 = bufferManager.fetchNewPageFrame(partNum2, false); + BufferFrame frame7 = bufferManager.fetchNewPageFrame(partNum2, false); + frame6.unpin(); + frame7.unpin(); + assertFalse(frame1.isValid()); + assertTrue(frame2.isValid()); + assertFalse(frame3.isValid()); + assertTrue(frame4.isValid()); + assertTrue(frame5.isValid()); + assertTrue(frame6.isValid()); + assertTrue(frame7.isValid()); + } + + @Test(expected = PageException.class) + public void testMissingPart() { + bufferManager.fetchPageFrame(DiskSpaceManager.getVirtualPageNum(0, 0), false); + } + + @Test(expected = PageException.class) + public void testMissingPage() { + int partNum = diskSpaceManager.allocPart(); + bufferManager.fetchPageFrame(DiskSpaceManager.getVirtualPageNum(partNum, 0), false); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/memory/TestEvictionPolicy.java b/src/test/java/edu/berkeley/cs186/database/memory/TestEvictionPolicy.java new file mode 100644 index 0000000..9537c48 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/memory/TestEvictionPolicy.java @@ -0,0 +1,196 @@ +package edu.berkeley.cs186.database.memory; + +import edu.berkeley.cs186.database.categories.HW99Tests; +import edu.berkeley.cs186.database.categories.SystemTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestEvictionPolicy { + private BufferFrame[] frames; + private BufferFrame[] placeholderFrames; + + private class TestFrame extends BufferFrame { + private int index; + private TestFrame(int index) { + this.index = index; + } + + @Override + public String toString() { + return "Frame #" + index; + } + + @Override + boolean isValid() { + return false; + } + + @Override + long getPageNum() { + return 0; + } + + @Override + void flush() { + } + + @Override + void readBytes(short position, short num, byte[] buf) { + } + + @Override + void writeBytes(short position, short num, byte[] buf) { + } + + @Override + long getPageLSN() { + return 0; + } + + @Override + void setPageLSN(long pageLSN) { + } + + @Override + BufferFrame requestValidFrame() { + return null; + } + } + + @Before + public void beforeEach() { + this.frames = new BufferFrame[8]; + this.placeholderFrames = new BufferFrame[8]; + for (int i = 0; i < this.frames.length; ++i) { + this.frames[i] = new TestFrame(i); + this.placeholderFrames[i] = new TestFrame(-i); + this.placeholderFrames[i].pin(); + } + } + + @Test + public void testLRUPolicy() { + EvictionPolicy policy = new LRUEvictionPolicy(); + policy.init(frames[0]); policy.hit(frames[0]); + policy.init(frames[1]); policy.hit(frames[1]); + policy.init(frames[2]); policy.hit(frames[2]); + policy.init(frames[3]); policy.hit(frames[3]); + + assertEquals(frames[0], policy.evict(new BufferFrame[] {frames[0], frames[1], frames[2], frames[3]})); + policy.cleanup(frames[0]); + + policy.init(frames[4]); policy.hit(frames[4]); + policy.hit(frames[1]); + frames[3].pin(); + + assertEquals(frames[2], policy.evict(new BufferFrame[] {frames[4], frames[1], frames[2], frames[3]})); + policy.cleanup(frames[2]); + + policy.init(frames[5]); policy.hit(frames[5]); + + assertEquals(frames[4], policy.evict(new BufferFrame[] {frames[4], frames[1], frames[5], frames[3]})); + policy.cleanup(frames[4]); + + policy.init(frames[6]); policy.hit(frames[6]); + + frames[3].unpin(); + + assertEquals(frames[3], policy.evict(new BufferFrame[] {frames[6], frames[1], frames[5], frames[3]})); + policy.cleanup(frames[3]); + + policy.init(frames[7]); policy.hit(frames[7]); + + policy.hit(frames[5]); + + assertEquals(frames[1], policy.evict(new BufferFrame[] {frames[6], frames[1], frames[5], frames[7]})); + policy.cleanup(frames[1]); + + policy.init(frames[3]); policy.hit(frames[3]); + + frames[3].pin(); + + assertEquals(frames[6], policy.evict(new BufferFrame[] {frames[6], frames[3], frames[5], frames[7]})); + policy.cleanup(frames[6]); + assertEquals(frames[7], policy.evict(new BufferFrame[] {placeholderFrames[0], frames[3], frames[5], frames[7]})); + policy.cleanup(frames[7]); + assertEquals(frames[5], policy.evict(new BufferFrame[] {placeholderFrames[0], frames[3], frames[5], placeholderFrames[3]})); + policy.cleanup(frames[5]); + boolean exceptionThrown = false; + try { + policy.evict(new BufferFrame[] {placeholderFrames[0], frames[3], placeholderFrames[2], placeholderFrames[3]}); + } catch (IllegalStateException e) { + exceptionThrown = true; + } + assertTrue(exceptionThrown); + + frames[3].unpin(); + assertEquals(frames[3], policy.evict(new BufferFrame[] {placeholderFrames[0], frames[3], placeholderFrames[2], placeholderFrames[3]})); + policy.cleanup(frames[3]); + } + + @Test + public void testClockPolicy() { + EvictionPolicy policy = new ClockEvictionPolicy(); + policy.init(frames[0]); policy.hit(frames[0]); + policy.init(frames[1]); policy.hit(frames[1]); + policy.init(frames[2]); policy.hit(frames[2]); + policy.init(frames[3]); policy.hit(frames[3]); + + assertEquals(frames[0], policy.evict(new BufferFrame[] {frames[0], frames[1], frames[2], frames[3]})); + policy.cleanup(frames[0]); + + policy.init(frames[4]); policy.hit(frames[4]); + + policy.hit(frames[1]); + frames[3].pin(); + + assertEquals(frames[2], policy.evict(new BufferFrame[] {frames[4], frames[1], frames[2], frames[3]})); + policy.cleanup(frames[2]); + + policy.init(frames[5]); policy.hit(frames[5]); + + assertEquals(frames[1], policy.evict(new BufferFrame[] {frames[4], frames[1], frames[5], frames[3]})); + policy.cleanup(frames[1]); + + policy.init(frames[6]); policy.hit(frames[6]); + + frames[3].unpin(); + + assertEquals(frames[3], policy.evict(new BufferFrame[] {frames[4], frames[6], frames[5], frames[3]})); + policy.cleanup(frames[3]); + + policy.init(frames[7]); policy.hit(frames[7]); + + policy.hit(frames[4]); + + assertEquals(frames[5], policy.evict(new BufferFrame[] {frames[4], frames[6], frames[5], frames[7]})); + policy.cleanup(frames[5]); + + policy.init(frames[2]); policy.hit(frames[2]); + + frames[2].pin(); + + assertEquals(frames[4], policy.evict(new BufferFrame[] {frames[4], frames[6], frames[2], frames[7]})); + policy.cleanup(frames[4]); + assertEquals(frames[6], policy.evict(new BufferFrame[] {placeholderFrames[0], frames[6], frames[2], frames[7]})); + policy.cleanup(frames[6]); + assertEquals(frames[7], policy.evict(new BufferFrame[] {placeholderFrames[0], placeholderFrames[1], frames[2], frames[7]})); + policy.cleanup(frames[7]); + boolean exceptionThrown = false; + try { + policy.evict(new BufferFrame[] {placeholderFrames[0], placeholderFrames[1], frames[2], placeholderFrames[3]}); + } catch (IllegalStateException e) { + exceptionThrown = true; + } + assertTrue(exceptionThrown); + + frames[2].unpin(); + assertEquals(frames[2], policy.evict(new BufferFrame[] {placeholderFrames[0], placeholderFrames[1], frames[2], placeholderFrames[3]})); + policy.cleanup(frames[2]); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/query/TestBasicQuery.java b/src/test/java/edu/berkeley/cs186/database/query/TestBasicQuery.java new file mode 100644 index 0000000..6ec72a3 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/query/TestBasicQuery.java @@ -0,0 +1,181 @@ +package edu.berkeley.cs186.database.query; + +import edu.berkeley.cs186.database.*; +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.table.Schema; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.util.Collections; +import java.util.Iterator; + +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.BoolDataBox; + +import static org.junit.Assert.*; +import org.junit.After; + +import edu.berkeley.cs186.database.TimeoutScaling; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +@Category({HW3Tests.class, HW3Part2Tests.class}) +public class TestBasicQuery { + private static final String TABLENAME = "T"; + + private static final String TestDir = "testDatabase"; + private Database db; + + //Before every test you create a temporary table, after every test you close it + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + // 1 second max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + @Before + public void beforeEach() throws Exception { + File testDir = tempFolder.newFolder(TestDir); + String filename = testDir.getAbsolutePath(); + this.db = new Database(filename, 32); + this.db.setWorkMem(5); // B=5 + this.db.waitSetupFinished(); + + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + + Schema schema = TestUtils.createSchemaWithAllTypes(); + + t.createTable(schema, TABLENAME); + + t.createTable(schema, TABLENAME + "2"); + t.createIndex(TABLENAME + "2", "int", false); + } + this.db.waitAllTransactions(); + } + + @After + public void afterEach() { + this.db.waitAllTransactions(); + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + } + this.db.close(); + } + + //creates a record with all specified types + private static Record createRecordWithAllTypes(boolean a1, int a2, String a3, float a4) { + Record r = TestUtils.createRecordWithAllTypes(); + r.getValues().set(0, new BoolDataBox(a1)); + r.getValues().set(1, new IntDataBox(a2)); + r.getValues().set(2, new StringDataBox(a3, 1)); + r.getValues().set(3, new FloatDataBox(a4)); + return r; + } + + @Test + @Category(PublicTests.class) + public void testProject() { + try(Transaction transaction = this.db.beginTransaction()) { + //creates a 10 records int 0 to 9 + for (int i = 0; i < 10; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME, r.getValues()); + } + + //build the statistics on the table + transaction.getTransactionContext().getTable(TABLENAME).buildStatistics(10); + + // add a project to the QueryPlan + QueryPlan query = transaction.query("T"); + query.project(Collections.singletonList("int")); + + // execute the query and get the output + Iterator queryOutput = query.execute(); + + query.getFinalOperator(); + + //tests to see if projects are applied properly + int count = 0; + while (queryOutput.hasNext()) { + Record r = queryOutput.next(); + assertEquals(r.getValues().get(0), new IntDataBox(count)); + count++; + } + } + } + + @Test + @Category(PublicTests.class) + public void testSelect() { + try(Transaction transaction = db.beginTransaction()) { + //creates a 10 records int 0 to 9 + for (int i = 0; i < 10; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME, r.getValues()); + } + + //build the statistics on the table + transaction.getTransactionContext().getTable(TABLENAME).buildStatistics(10); + + // add a select to the QueryPlan + QueryPlan query = transaction.query("T"); + query.select("int", PredicateOperator.EQUALS, new IntDataBox(9)); + + // execute the query and get the output + Iterator queryOutput = query.execute(); + + query.getFinalOperator(); + + //tests to see if projects are applied properly + assert (queryOutput.hasNext()); + + Record r = queryOutput.next(); + assertEquals(r.getValues().get(1), new IntDataBox(9)); + } + } + + @Test + @Category(PublicTests.class) + public void testGroupBy() { + try(Transaction transaction = db.beginTransaction()) { + //creates a 100 records int 0 to 9 + for (int i = 0; i < 100; ++i) { + Record r = createRecordWithAllTypes(false, i % 10, "!", 0.0f); + transaction.insert(TABLENAME, r.getValues()); + } + + //build the statistics on the table + transaction.getTransactionContext().getTable(TABLENAME).buildStatistics(10); + + // add a project and a groupby to the QueryPlan + QueryPlan query = transaction.query("T"); + query.groupBy("T.int"); + query.count(); + + // execute the query and get the output + Iterator queryOutput = query.execute(); + + query.getFinalOperator(); + + //tests to see if projects/group by are applied properly + int count = 0; + while (queryOutput.hasNext()) { + Record r = queryOutput.next(); + assertEquals(r.getValues().get(0).getInt(), 10); + count++; + } + + assertEquals(count, 10); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/query/TestJoinOperator.java b/src/test/java/edu/berkeley/cs186/database/query/TestJoinOperator.java new file mode 100644 index 0000000..6937161 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/query/TestJoinOperator.java @@ -0,0 +1,535 @@ +package edu.berkeley.cs186.database.query; + +import edu.berkeley.cs186.database.*; +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.Page; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import edu.berkeley.cs186.database.databox.BoolDataBox; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.table.Record; + +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import static org.junit.Assert.*; + +@Category({HW3Tests.class, HW3Part1Tests.class}) +public class TestJoinOperator { + private Database d; + private long numIOs; + private QueryOperator leftSourceOperator; + private QueryOperator rightSourceOperator; + private Map pinnedPages = new HashMap<>(); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setup() throws IOException { + File tempDir = tempFolder.newFolder("joinTest"); + d = new Database(tempDir.getAbsolutePath(), 256); + d.setWorkMem(5); // B=5 + d.waitAllTransactions(); + } + + @After + public void cleanup() { + for (Page p : pinnedPages.values()) { + p.unpin(); + } + d.close(); + } + + // 4 second max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 4000 * TimeoutScaling.factor))); + + private void startCountIOs() { + d.getBufferManager().evictAll(); + numIOs = d.getBufferManager().getNumIOs(); + } + + private void checkIOs(String message, long minIOs, long maxIOs) { + if (message == null) { + message = ""; + } else { + message = "(" + message + ")"; + } + + long newIOs = d.getBufferManager().getNumIOs(); + long IOs = newIOs - numIOs; + + assertTrue(IOs + " I/Os not between " + minIOs + " and " + maxIOs + message, + minIOs <= IOs && IOs <= maxIOs); + numIOs = newIOs; + } + + private void checkIOs(String message, long numIOs) { + checkIOs(message, numIOs, numIOs); + } + + private void checkIOs(long minIOs, long maxIOs) { + checkIOs(null, minIOs, maxIOs); + } + private void checkIOs(long numIOs) { + checkIOs(null, numIOs, numIOs); + } + + private void setSourceOperators(TestSourceOperator leftSourceOperator, + TestSourceOperator rightSourceOperator, Transaction transaction) { + setSourceOperators( + new MaterializeOperator(leftSourceOperator, transaction.getTransactionContext()), + new MaterializeOperator(rightSourceOperator, transaction.getTransactionContext()) + ); + } + + private void pinPage(int partNum, int pageNum) { + long pnum = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + Page page = d.getBufferManager().fetchPage(new DummyLockContext(), pnum, false); + this.pinnedPages.put(pnum, page); + } + + private void unpinPage(int partNum, int pageNum) { + long pnum = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + this.pinnedPages.remove(pnum).unpin(); + } + + private void evictPage(int partNum, int pageNum) { + long pnum = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + this.d.getBufferManager().evict(pnum); + numIOs = d.getBufferManager().getNumIOs(); + } + + private void setSourceOperators(QueryOperator leftSourceOperator, + QueryOperator rightSourceOperator) { + assert (this.leftSourceOperator == null && this.rightSourceOperator == null); + + this.leftSourceOperator = leftSourceOperator; + this.rightSourceOperator = rightSourceOperator; + + // hard-coded mess, but works as long as the first two tables created are the source operators + pinPage(1, 0); // information_schema.tables header page + pinPage(1, 3); // information_schema.tables entry for left source + pinPage(1, 4); // information_schema.tables entry for right source + pinPage(3, 0); // left source header page + pinPage(4, 0); // right source header page + } + + @Test + @Category(PublicTests.class) + public void testSimpleJoinPNLJ() { + try(Transaction transaction = d.beginTransaction()) { + setSourceOperators( + new TestSourceOperator(), + new TestSourceOperator(), + transaction + ); + + startCountIOs(); + + JoinOperator joinOperator = new PNLJOperator(leftSourceOperator, rightSourceOperator, "int", "int", + transaction.getTransactionContext()); + checkIOs(0); + + Iterator outputIterator = joinOperator.iterator(); + checkIOs(2); + + int numRecords = 0; + List expectedRecordValues = new ArrayList<>(); + expectedRecordValues.add(new BoolDataBox(true)); + expectedRecordValues.add(new IntDataBox(1)); + expectedRecordValues.add(new StringDataBox("a", 1)); + expectedRecordValues.add(new FloatDataBox(1.2f)); + expectedRecordValues.add(new BoolDataBox(true)); + expectedRecordValues.add(new IntDataBox(1)); + expectedRecordValues.add(new StringDataBox("a", 1)); + expectedRecordValues.add(new FloatDataBox(1.2f)); + Record expectedRecord = new Record(expectedRecordValues); + + while (outputIterator.hasNext() && numRecords < 100 * 100) { + assertEquals("mismatch at record " + numRecords, expectedRecord, outputIterator.next()); + numRecords++; + } + checkIOs(0); + + assertFalse("too many records", outputIterator.hasNext()); + assertEquals("too few records", 100 * 100, numRecords); + } + } + + @Test + @Category(PublicTests.class) + public void testSimpleJoinBNLJ() { + d.setWorkMem(5); // B=5 + try(Transaction transaction = d.beginTransaction()) { + setSourceOperators( + new TestSourceOperator(), + new TestSourceOperator(), + transaction + ); + + startCountIOs(); + + JoinOperator joinOperator = new BNLJOperator(leftSourceOperator, rightSourceOperator, "int", "int", + transaction.getTransactionContext()); + checkIOs(0); + + Iterator outputIterator = joinOperator.iterator(); + checkIOs(2); + + int numRecords = 0; + List expectedRecordValues = new ArrayList<>(); + expectedRecordValues.add(new BoolDataBox(true)); + expectedRecordValues.add(new IntDataBox(1)); + expectedRecordValues.add(new StringDataBox("a", 1)); + expectedRecordValues.add(new FloatDataBox(1.2f)); + expectedRecordValues.add(new BoolDataBox(true)); + expectedRecordValues.add(new IntDataBox(1)); + expectedRecordValues.add(new StringDataBox("a", 1)); + expectedRecordValues.add(new FloatDataBox(1.2f)); + Record expectedRecord = new Record(expectedRecordValues); + + while (outputIterator.hasNext() && numRecords < 100 * 100) { + assertEquals("mismatch at record " + numRecords, expectedRecord, outputIterator.next()); + numRecords++; + } + checkIOs(0); + + assertFalse("too many records", outputIterator.hasNext()); + assertEquals("too few records", 100 * 100, numRecords); + } + } + + @Test + @Category(PublicTests.class) + public void testSimplePNLJOutputOrder() { + try(Transaction transaction = d.beginTransaction()) { + Record r1 = TestUtils.createRecordWithAllTypesWithValue(1); + List r1Vals = r1.getValues(); + Record r2 = TestUtils.createRecordWithAllTypesWithValue(2); + List r2Vals = r2.getValues(); + + List expectedRecordValues1 = new ArrayList<>(); + List expectedRecordValues2 = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + expectedRecordValues1.addAll(r1Vals); + expectedRecordValues2.addAll(r2Vals); + } + + Record expectedRecord1 = new Record(expectedRecordValues1); + Record expectedRecord2 = new Record(expectedRecordValues2); + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "leftTable"); + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "rightTable"); + + for (int i = 0; i < 400; i++) { + List vals; + if (i < 200) { + vals = r1Vals; + } else { + vals = r2Vals; + } + transaction.getTransactionContext().addRecord("leftTable", vals); + transaction.getTransactionContext().addRecord("rightTable", vals); + } + + for (int i = 0; i < 400; i++) { + if (i < 200) { + transaction.getTransactionContext().addRecord("leftTable", r2Vals); + transaction.getTransactionContext().addRecord("rightTable", r1Vals); + } else { + transaction.getTransactionContext().addRecord("leftTable", r1Vals); + transaction.getTransactionContext().addRecord("rightTable", r2Vals); + } + } + + setSourceOperators( + new SequentialScanOperator(transaction.getTransactionContext(), "leftTable"), + new SequentialScanOperator(transaction.getTransactionContext(), "rightTable") + ); + + startCountIOs(); + + QueryOperator joinOperator = new PNLJOperator(leftSourceOperator, rightSourceOperator, "int", "int", + transaction.getTransactionContext()); + checkIOs(0); + + int count = 0; + Iterator outputIterator = joinOperator.iterator(); + checkIOs(2); + + while (outputIterator.hasNext() && count < 400 * 400 * 2) { + if (count < 200 * 200) { + assertEquals("mismatch at record " + count, expectedRecord1, outputIterator.next()); + } else if (count < 200 * 200 * 2) { + assertEquals("mismatch at record " + count, expectedRecord2, outputIterator.next()); + } else if (count < 200 * 200 * 3) { + assertEquals("mismatch at record " + count, expectedRecord1, outputIterator.next()); + } else if (count < 200 * 200 * 4) { + assertEquals("mismatch at record " + count, expectedRecord2, outputIterator.next()); + } else if (count < 200 * 200 * 5) { + assertEquals("mismatch at record " + count, expectedRecord2, outputIterator.next()); + } else if (count < 200 * 200 * 6) { + assertEquals("mismatch at record " + count, expectedRecord1, outputIterator.next()); + } else if (count < 200 * 200 * 7) { + assertEquals("mismatch at record " + count, expectedRecord2, outputIterator.next()); + } else { + assertEquals("mismatch at record " + count, expectedRecord1, outputIterator.next()); + } + count++; + + if (count == 200 * 200 * 2 || count == 200 * 200 * 6) { + checkIOs("at record " + count, 1); + evictPage(4, 1); + } else if (count == 200 * 200 * 4) { + checkIOs("at record " + count, 2); + evictPage(4, 2); + evictPage(3, 1); + } else { + checkIOs("at record " + count, 0); + } + } + + assertFalse("too many records", outputIterator.hasNext()); + assertEquals("too few records", 400 * 400 * 2, count); + } + } + + @Test + @Category(PublicTests.class) + public void testSimpleSortMergeJoin() { + d.setWorkMem(5); // B=5 + try(Transaction transaction = d.beginTransaction()) { + setSourceOperators( + new TestSourceOperator(), + new TestSourceOperator(), + transaction + ); + + startCountIOs(); + + JoinOperator joinOperator = new SortMergeOperator(leftSourceOperator, rightSourceOperator, "int", + "int", + transaction.getTransactionContext()); + checkIOs(0); + + Iterator outputIterator = joinOperator.iterator(); + checkIOs(2 * (1 + (1 + TestSortOperator.NEW_TABLE_IOS))); + + int numRecords = 0; + List expectedRecordValues = new ArrayList<>(); + expectedRecordValues.add(new BoolDataBox(true)); + expectedRecordValues.add(new IntDataBox(1)); + expectedRecordValues.add(new StringDataBox("a", 1)); + expectedRecordValues.add(new FloatDataBox(1.2f)); + expectedRecordValues.add(new BoolDataBox(true)); + expectedRecordValues.add(new IntDataBox(1)); + expectedRecordValues.add(new StringDataBox("a", 1)); + expectedRecordValues.add(new FloatDataBox(1.2f)); + Record expectedRecord = new Record(expectedRecordValues); + + while (outputIterator.hasNext() && numRecords < 100 * 100) { + assertEquals("mismatch at record " + numRecords, expectedRecord, outputIterator.next()); + numRecords++; + } + checkIOs(0); + + assertFalse("too many records", outputIterator.hasNext()); + assertEquals("too few records", 100 * 100, numRecords); + } + } + + @Test + @Category(PublicTests.class) + public void testSortMergeJoinUnsortedInputs() { + d.setWorkMem(3); // B=3 + try(Transaction transaction = d.beginTransaction()) { + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "leftTable"); + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "rightTable"); + Record r1 = TestUtils.createRecordWithAllTypesWithValue(1); + List r1Vals = r1.getValues(); + Record r2 = TestUtils.createRecordWithAllTypesWithValue(2); + List r2Vals = r2.getValues(); + Record r3 = TestUtils.createRecordWithAllTypesWithValue(3); + List r3Vals = r3.getValues(); + Record r4 = TestUtils.createRecordWithAllTypesWithValue(4); + List r4Vals = r4.getValues(); + List expectedRecordValues1 = new ArrayList<>(); + List expectedRecordValues2 = new ArrayList<>(); + List expectedRecordValues3 = new ArrayList<>(); + List expectedRecordValues4 = new ArrayList<>(); + + for (int i = 0; i < 2; i++) { + expectedRecordValues1.addAll(r1Vals); + expectedRecordValues2.addAll(r2Vals); + expectedRecordValues3.addAll(r3Vals); + expectedRecordValues4.addAll(r4Vals); + } + Record expectedRecord1 = new Record(expectedRecordValues1); + Record expectedRecord2 = new Record(expectedRecordValues2); + Record expectedRecord3 = new Record(expectedRecordValues3); + Record expectedRecord4 = new Record(expectedRecordValues4); + List leftTableRecords = new ArrayList<>(); + List rightTableRecords = new ArrayList<>(); + for (int i = 0; i < 400 * 2; i++) { + Record r; + if (i % 4 == 0) { + r = r1; + } else if (i % 4 == 1) { + r = r2; + } else if (i % 4 == 2) { + r = r3; + } else { + r = r4; + } + leftTableRecords.add(r); + rightTableRecords.add(r); + } + Collections.shuffle(leftTableRecords, new Random(10)); + Collections.shuffle(rightTableRecords, new Random(20)); + for (int i = 0; i < 400 * 2; i++) { + transaction.getTransactionContext().addRecord("leftTable", leftTableRecords.get(i).getValues()); + transaction.getTransactionContext().addRecord("rightTable", rightTableRecords.get(i).getValues()); + } + + setSourceOperators( + new SequentialScanOperator(transaction.getTransactionContext(), "leftTable"), + new SequentialScanOperator(transaction.getTransactionContext(), "rightTable") + ); + + startCountIOs(); + + JoinOperator joinOperator = new SortMergeOperator(leftSourceOperator, rightSourceOperator, "int", + "int", + transaction.getTransactionContext()); + checkIOs(0); + + Iterator outputIterator = joinOperator.iterator(); + checkIOs(2 * (2 + (2 + TestSortOperator.NEW_TABLE_IOS))); + + int numRecords = 0; + Record expectedRecord; + + while (outputIterator.hasNext() && numRecords < 400 * 400) { + if (numRecords < (400 * 400 / 4)) { + expectedRecord = expectedRecord1; + } else if (numRecords < (400 * 400 / 2)) { + expectedRecord = expectedRecord2; + } else if (numRecords < 400 * 400 - (400 * 400 / 4)) { + expectedRecord = expectedRecord3; + } else { + expectedRecord = expectedRecord4; + } + Record r = outputIterator.next(); + assertEquals("mismatch at record " + numRecords, r, expectedRecord); + numRecords++; + } + checkIOs(0); + + assertFalse("too many records", outputIterator.hasNext()); + assertEquals("too few records", 400 * 400, numRecords); + } + } + + @Test + @Category(PublicTests.class) + public void testBNLJDiffOutPutThanPNLJ() { + d.setWorkMem(4); // B=4 + try(Transaction transaction = d.beginTransaction()) { + Record r1 = TestUtils.createRecordWithAllTypesWithValue(1); + List r1Vals = r1.getValues(); + Record r2 = TestUtils.createRecordWithAllTypesWithValue(2); + List r2Vals = r2.getValues(); + Record r3 = TestUtils.createRecordWithAllTypesWithValue(3); + List r3Vals = r3.getValues(); + Record r4 = TestUtils.createRecordWithAllTypesWithValue(4); + List r4Vals = r4.getValues(); + List expectedRecordValues1 = new ArrayList<>(); + List expectedRecordValues2 = new ArrayList<>(); + List expectedRecordValues3 = new ArrayList<>(); + List expectedRecordValues4 = new ArrayList<>(); + + for (int i = 0; i < 2; i++) { + expectedRecordValues1.addAll(r1Vals); + expectedRecordValues2.addAll(r2Vals); + expectedRecordValues3.addAll(r3Vals); + expectedRecordValues4.addAll(r4Vals); + } + Record expectedRecord1 = new Record(expectedRecordValues1); + Record expectedRecord2 = new Record(expectedRecordValues2); + Record expectedRecord3 = new Record(expectedRecordValues3); + Record expectedRecord4 = new Record(expectedRecordValues4); + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "leftTable"); + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "rightTable"); + for (int i = 0; i < 2 * 400; i++) { + if (i < 200) { + transaction.getTransactionContext().addRecord("leftTable", r1Vals); + transaction.getTransactionContext().addRecord("rightTable", r3Vals); + } else if (i < 400) { + transaction.getTransactionContext().addRecord("leftTable", r2Vals); + transaction.getTransactionContext().addRecord("rightTable", r4Vals); + } else if (i < 600) { + transaction.getTransactionContext().addRecord("leftTable", r3Vals); + transaction.getTransactionContext().addRecord("rightTable", r1Vals); + } else { + transaction.getTransactionContext().addRecord("leftTable", r4Vals); + transaction.getTransactionContext().addRecord("rightTable", r2Vals); + } + } + + setSourceOperators( + new SequentialScanOperator(transaction.getTransactionContext(), "leftTable"), + new SequentialScanOperator(transaction.getTransactionContext(), "rightTable") + ); + + startCountIOs(); + + QueryOperator joinOperator = new BNLJOperator(leftSourceOperator, rightSourceOperator, "int", "int", + transaction.getTransactionContext()); + checkIOs(0); + + Iterator outputIterator = joinOperator.iterator(); + checkIOs(3); + + int count = 0; + while (outputIterator.hasNext() && count < 4 * 200 * 200) { + Record r = outputIterator.next(); + if (count < 200 * 200) { + assertEquals("mismatch at record " + count, expectedRecord3, r); + } else if (count < 2 * 200 * 200) { + assertEquals("mismatch at record " + count, expectedRecord4, r); + } else if (count < 3 * 200 * 200) { + assertEquals("mismatch at record " + count, expectedRecord1, r); + } else { + assertEquals("mismatch at record " + count, expectedRecord2, r); + } + + count++; + + if (count == 200 * 200 * 2) { + checkIOs("at record " + count, 1); + } else { + checkIOs("at record " + count, 0); + } + } + assertFalse("too many records", outputIterator.hasNext()); + assertEquals("too few records", 4 * 200 * 200, count); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/query/TestOptimization2.java b/src/test/java/edu/berkeley/cs186/database/query/TestOptimization2.java new file mode 100644 index 0000000..581113a --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/query/TestOptimization2.java @@ -0,0 +1,104 @@ +package edu.berkeley.cs186.database.query; + +import edu.berkeley.cs186.database.*; +import edu.berkeley.cs186.database.categories.*; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +import edu.berkeley.cs186.database.table.Schema; + +import edu.berkeley.cs186.database.table.Table; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.BoolDataBox; + +import org.junit.After; + +import edu.berkeley.cs186.database.TimeoutScaling; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +@Category({HW3Tests.class, HW3Part2Tests.class}) +public class TestOptimization2 { + private static final String TABLENAME = "T"; + + private static final String TestDir = "testDatabase"; + private Database db; + + //Before every test you create a temporary table, after every test you close it + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + // 1 second max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + @Before + public void beforeEach() throws Exception { + File testDir = tempFolder.newFolder(TestDir); + String filename = testDir.getAbsolutePath(); + this.db = new Database(filename, 32); + this.db.setWorkMem(5); // B=5 + + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + + Schema schema = TestUtils.createSchemaWithAllTypes(); + + t.createTable(schema, TABLENAME); + + //t.createTableWithIndices(schema, TABLENAME, Arrays.asList("int")); + } + this.db.waitAllTransactions(); + } + + @After + public void afterEach() { + this.db.waitAllTransactions(); + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + } + this.db.close(); + } + + //creates a record with all specified types + private static Record createRecordWithAllTypes(boolean a1, int a2, String a3, float a4) { + Record r = TestUtils.createRecordWithAllTypes(); + r.getValues().set(0, new BoolDataBox(a1)); + r.getValues().set(1, new IntDataBox(a2)); + r.getValues().set(2, new StringDataBox(a3, 1)); + r.getValues().set(3, new FloatDataBox(a4)); + return r; + } + + @Test + @Category(PublicTests.class) + public void test() { + try(Transaction transaction = this.db.beginTransaction()) { + //creates a 100 records int 0 to 99 + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME, r.getValues()); + } + + //build the statistics on the table + transaction.getTransactionContext().getTable(TABLENAME).buildStatistics(10); + + // add a join and a select to the QueryPlan + QueryPlan query = transaction.query("T", "t1"); + query.join("T", "t2", "t1.int", "t2.int"); + //query.select("int", PredicateOperator.EQUALS, new IntDataBox(10)); + + // execute the query and get the output + query.execute(); + query.getFinalOperator(); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/query/TestOptimizationJoins.java b/src/test/java/edu/berkeley/cs186/database/query/TestOptimizationJoins.java new file mode 100644 index 0000000..f2c1802 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/query/TestOptimizationJoins.java @@ -0,0 +1,250 @@ +package edu.berkeley.cs186.database.query; + +import edu.berkeley.cs186.database.*; +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.PredicateOperator; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +import edu.berkeley.cs186.database.table.Schema; + +import edu.berkeley.cs186.database.table.Table; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.BoolDataBox; + +import org.junit.After; + +import edu.berkeley.cs186.database.TimeoutScaling; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@Category({HW3Tests.class, HW3Part2Tests.class}) +public class TestOptimizationJoins { + private static final String TABLENAME = "T"; + + private static final String TestDir = "testDatabase"; + private Database db; + + //Before every test you create a temporary table, after every test you close it + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + // 10 second max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 10000 * TimeoutScaling.factor))); + + @Before + public void beforeEach() throws Exception { + File testDir = tempFolder.newFolder(TestDir); + String filename = testDir.getAbsolutePath(); + this.db = new Database(filename, 32); + this.db.setWorkMem(5); // B=5 + this.db.waitSetupFinished(); + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + + Schema schema = TestUtils.createSchemaWithAllTypes(); + + t.createTable(schema, TABLENAME); + + t.createTable(schema, TABLENAME + "I"); + t.createIndex(TABLENAME + "I", "int", false); + + t.createTable(TestUtils.createSchemaWithAllTypes("one_"), TABLENAME + "o1"); + t.createTable(TestUtils.createSchemaWithAllTypes("two_"), TABLENAME + "o2"); + t.createTable(TestUtils.createSchemaWithAllTypes("three_"), TABLENAME + "o3"); + t.createTable(TestUtils.createSchemaWithAllTypes("four_"), TABLENAME + "o4"); + } + this.db.waitAllTransactions(); + } + + @After + public void afterEach() { + this.db.waitAllTransactions(); + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + } + this.db.close(); + } + + //creates a record with all specified types + private static Record createRecordWithAllTypes(boolean a1, int a2, String a3, float a4) { + Record r = TestUtils.createRecordWithAllTypes(); + r.getValues().set(0, new BoolDataBox(a1)); + r.getValues().set(1, new IntDataBox(a2)); + r.getValues().set(2, new StringDataBox(a3, 1)); + r.getValues().set(3, new FloatDataBox(a4)); + return r; + } + + @Test + @Category(PublicTests.class) + public void testJoinTypeA() { + try(Transaction transaction = this.db.beginTransaction()) { + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME, r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME).buildStatistics(10); + + // add a join and a select to the QueryPlan + QueryPlan query = transaction.query("T", "t1"); + query.join("T", "t2", "t1.int", "t2.int"); + + //query.select("int", PredicateOperator.EQUALS, new IntDataBox(10)); + + // execute the query + query.execute(); + + QueryOperator finalOperator = query.getFinalOperator(); + assertTrue(finalOperator.toString().contains("BNLJ")); + } + } + + @Test + @Category(PublicTests.class) + public void testJoinTypeB() { + try(Transaction transaction = this.db.beginTransaction()) { + for (int i = 0; i < 10; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "I", r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME + "I").buildStatistics(10); + + // add a join and a select to the QueryPlan + QueryPlan query = transaction.query("TI", "t1"); + query.join("TI", "t2", "t1.int", "t2.int"); + query.select("int", PredicateOperator.EQUALS, new IntDataBox(9)); + + // execute the query + query.execute(); + + QueryOperator finalOperator = query.getFinalOperator(); + + assertTrue(finalOperator.toString().contains("\tvalue: 9")); + } + } + + @Test + @Category(PublicTests.class) + public void testJoinTypeC() { + try(Transaction transaction = db.beginTransaction()) { + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "I", r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME + "I").buildStatistics(10); + + // add a join and a select to the QueryPlan + QueryPlan query = transaction.query("TI", "t1"); + query.join("TI", "t2", "t1.int", "t2.int"); + query.select("int", PredicateOperator.EQUALS, new IntDataBox(9)); + + // execute the query + query.execute(); + + QueryOperator finalOperator = query.getFinalOperator(); + + assertTrue(finalOperator.toString().contains("INDEXSCAN")); + + assertTrue(finalOperator.toString().contains("SNLJ")); + } + } + + @Test + @Category(PublicTests.class) + public void testJoinOrderA() { + try(Transaction transaction = db.beginTransaction()) { + for (int i = 0; i < 10; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "o1", r.getValues()); + } + + for (int i = 0; i < 100; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "o2", r.getValues()); + } + + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "o3", r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME + "o1").buildStatistics(10); + transaction.getTransactionContext().getTable(TABLENAME + "o2").buildStatistics(10); + transaction.getTransactionContext().getTable(TABLENAME + "o3").buildStatistics(10); + + // add a join and a select to the QueryPlan + QueryPlan query = transaction.query("To1"); + query.join("To2", "To1.one_int", "To2.two_int"); + query.join("To3", "To2.two_int", "To3.three_int"); + + // execute the query + query.execute(); + + QueryOperator finalOperator = query.getFinalOperator(); + //inner most joins are the largest tables + assertTrue(finalOperator.toString().contains("\t\ttable: To2")); + assertTrue(finalOperator.toString().contains("\t\ttable: To3")); + } + } + + @Test + @Category(PublicTests.class) + public void testJoinOrderB() { + try(Transaction transaction = db.beginTransaction()) { + for (int i = 0; i < 10; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "o1", r.getValues()); + } + + for (int i = 0; i < 100; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "o2", r.getValues()); + } + + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "o3", r.getValues()); + transaction.insert(TABLENAME + "o4", r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME + "o1").buildStatistics(10); + transaction.getTransactionContext().getTable(TABLENAME + "o2").buildStatistics(10); + transaction.getTransactionContext().getTable(TABLENAME + "o3").buildStatistics(10); + transaction.getTransactionContext().getTable(TABLENAME + "o4").buildStatistics(10); + + // add a join and a select to the QueryPlan + QueryPlan query = transaction.query("To1"); + query.join("To2", "To1.one_int", "To2.two_int"); + query.join("To3", "To2.two_int", "To3.three_int"); + query.join("To4", "To1.one_string", "To4.four_string"); + + // execute the query + query.execute(); + + QueryOperator finalOperator = query.getFinalOperator(); + + //smallest to largest order + assertTrue(finalOperator.toString().contains("\t\t\ttable: To2")); + assertTrue(finalOperator.toString().contains("\t\t\ttable: To3")); + assertTrue(finalOperator.toString().contains("\t\ttable: To1")); + assertTrue(finalOperator.toString().contains("\ttable: To4")); + } + } + +} diff --git a/src/test/java/edu/berkeley/cs186/database/query/TestSingleAccess.java b/src/test/java/edu/berkeley/cs186/database/query/TestSingleAccess.java new file mode 100644 index 0000000..2cb8fd2 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/query/TestSingleAccess.java @@ -0,0 +1,215 @@ +package edu.berkeley.cs186.database.query; + +import edu.berkeley.cs186.database.*; +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.PredicateOperator; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.util.Collections; + +import edu.berkeley.cs186.database.table.Schema; + +import edu.berkeley.cs186.database.table.Table; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.BoolDataBox; + +import edu.berkeley.cs186.database.TimeoutScaling; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import static org.junit.Assert.assertTrue; + +@Category({HW3Tests.class, HW3Part2Tests.class}) +public class TestSingleAccess { + private static final String TABLENAME = "T"; + + private static final String TestDir = "testDatabase"; + private Database db; + + //Before every test you create a temporary table, after every test you close it + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + // 2 second max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 2000 * TimeoutScaling.factor))); + + @Before + public void beforeEach() throws Exception { + File testDir = tempFolder.newFolder(TestDir); + String filename = testDir.getAbsolutePath(); + this.db = new Database(filename, 32); + this.db.setWorkMem(5); // B=5 + this.db.waitSetupFinished(); + + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + + Schema schema = TestUtils.createSchemaWithAllTypes(); + + t.createTable(schema, TABLENAME); + + t.createTable(schema, TABLENAME + "I"); + t.createIndex(TABLENAME + "I", "int", false); + t.createTable(schema, TABLENAME + "MI"); + t.createIndex(TABLENAME + "MI", "int", false); + t.createIndex(TABLENAME + "MI", "float", false); + + t.createTable(TestUtils.createSchemaWithAllTypes("one_"), TABLENAME + "o1"); + t.createTable(TestUtils.createSchemaWithAllTypes("two_"), TABLENAME + "o2"); + t.createTable(TestUtils.createSchemaWithAllTypes("three_"), TABLENAME + "o3"); + t.createTable(TestUtils.createSchemaWithAllTypes("four_"), TABLENAME + "o4"); + } + this.db.waitAllTransactions(); + } + + @After + public void afterEach() { + this.db.waitAllTransactions(); + try(Transaction t = this.db.beginTransaction()) { + t.dropAllTables(); + } + this.db.close(); + } + + //creates a record with all specified types + private static Record createRecordWithAllTypes(boolean a1, int a2, String a3, float a4) { + Record r = TestUtils.createRecordWithAllTypes(); + r.getValues().set(0, new BoolDataBox(a1)); + r.getValues().set(1, new IntDataBox(a2)); + r.getValues().set(2, new StringDataBox(a3, 1)); + r.getValues().set(3, new FloatDataBox(a4)); + return r; + } + + @Test + @Category(PublicTests.class) + public void testSequentialScanSelection() { + try(Transaction transaction = this.db.beginTransaction()) { + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME, r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME).buildStatistics(10); + + QueryPlan query = transaction.query(TABLENAME, "t1"); + + QueryOperator op = query.minCostSingleAccess("t1"); + + assertTrue(op.isSequentialScan()); + } + } + + @Test + @Category(PublicTests.class) + public void testSimpleIndexScanSelection() { + try(Transaction transaction = this.db.beginTransaction()) { + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME + "I", r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME + "I").buildStatistics(10); + + QueryPlan query = transaction.query(TABLENAME + "I", "t1"); + query.select("int", PredicateOperator.EQUALS, new IntDataBox(9)); + + QueryOperator op = query.minCostSingleAccess("t1"); + + assertTrue(op.isIndexScan()); + } + } + + @Test + @Category(PublicTests.class) + public void testPushDownSelects() { + try(Transaction transaction = this.db.beginTransaction()) { + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME, r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME).buildStatistics(10); + + QueryPlan query = transaction.query(TABLENAME, "t1"); + query.select("int", PredicateOperator.EQUALS, new IntDataBox(9)); + + QueryOperator op = query.minCostSingleAccess("t1"); + + assertTrue(op.isSelect()); + assertTrue(op.getSource().isSequentialScan()); + } + } + + @Test + @Category(PublicTests.class) + public void testPushDownMultipleSelects() { + try(Transaction transaction = this.db.beginTransaction()) { + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + transaction.insert(TABLENAME, r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME).buildStatistics(10); + + QueryPlan query = transaction.query(TABLENAME, "t1"); + query.select("int", PredicateOperator.EQUALS, new IntDataBox(9)); + query.select("bool", PredicateOperator.EQUALS, new BoolDataBox(false)); + + QueryOperator op = query.minCostSingleAccess("t1"); + + assertTrue(op.isSelect()); + assertTrue(op.getSource().isSelect()); + assertTrue(op.getSource().getSource().isSequentialScan()); + } + } + + @Test + @Category(PublicTests.class) + public void testNoValidIndices() { + try(Transaction transaction = this.db.beginTransaction()) { + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", i); + transaction.insert(TABLENAME + "MI", r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME + "MI").buildStatistics(10); + + QueryPlan query = transaction.query(TABLENAME + "MI", "t1"); + + QueryOperator op = query.minCostSingleAccess("t1"); + + assertTrue(op.isSequentialScan()); + } + } + + @Test + @Category(PublicTests.class) + public void testIndexSelectionAndPushDown() { + try(Transaction transaction = this.db.beginTransaction()) { + for (int i = 0; i < 2000; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", i); + transaction.insert(TABLENAME + "MI", r.getValues()); + } + + transaction.getTransactionContext().getTable(TABLENAME + "MI").buildStatistics(10); + QueryPlan query = transaction.query(TABLENAME + "MI", "t1"); + query.select("int", PredicateOperator.EQUALS, new IntDataBox(9)); + query.select("bool", PredicateOperator.EQUALS, new BoolDataBox(false)); + + QueryOperator op = query.minCostSingleAccess("t1"); + + assertTrue(op.isSelect()); + assertTrue(op.getSource().isIndexScan()); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/query/TestSortOperator.java b/src/test/java/edu/berkeley/cs186/database/query/TestSortOperator.java new file mode 100644 index 0000000..2895475 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/query/TestSortOperator.java @@ -0,0 +1,394 @@ +package edu.berkeley.cs186.database.query; + +import com.sun.org.apache.bcel.internal.generic.NEW; +import edu.berkeley.cs186.database.*; +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.memory.Page; +import org.junit.*; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import edu.berkeley.cs186.database.table.Record; + +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import static org.junit.Assert.*; + +@Category({HW3Tests.class, HW3Part1Tests.class}) +public class TestSortOperator { + private Database d; + private long numIOs; + private Map pinnedPages = new HashMap<>(); + + public static long FIRST_ACCESS_IOS = 1; // 1 I/O on first access to a table (after evictAll) + public static long NEW_TABLE_IOS = 2; // 2 I/Os to create a new table + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + // 2 second max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 2000 * TimeoutScaling.factor))); + + @Ignore + public static class SortRecordComparator implements Comparator { + private int columnIndex; + + private SortRecordComparator(int columnIndex) { + this.columnIndex = columnIndex; + } + @Override + public int compare(Record o1, Record o2) { + return o1.getValues().get(this.columnIndex).compareTo( + o2.getValues().get(this.columnIndex)); + } + } + + @Before + public void setup() throws IOException { + File tempDir = tempFolder.newFolder("sortTest"); + d = new Database(tempDir.getAbsolutePath(), 256); + d.setWorkMem(3); // B=3 + d.waitAllTransactions(); + } + + @After + public void cleanup() { + d.waitAllTransactions(); + for (Page p : pinnedPages.values()) { + p.unpin(); + } + d.close(); + } + + private void startCountIOs() { + d.getBufferManager().evictAll(); + numIOs = d.getBufferManager().getNumIOs(); + } + + private void checkIOs(String message, long minIOs, long maxIOs) { + if (message == null) { + message = ""; + } else { + message = "(" + message + ")"; + } + + long newIOs = d.getBufferManager().getNumIOs(); + long IOs = newIOs - numIOs; + + assertTrue(IOs + " I/Os not between " + minIOs + " and " + maxIOs + message, + minIOs <= IOs && IOs <= maxIOs); + numIOs = newIOs; + } + + private void checkIOs(String message, long numIOs) { + checkIOs(message, numIOs, numIOs); + } + + private void checkIOs(long minIOs, long maxIOs) { + checkIOs(null, minIOs, maxIOs); + } + private void checkIOs(long numIOs) { + checkIOs(null, numIOs, numIOs); + } + + private void pinPage(int partNum, int pageNum) { + long pnum = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + Page page = d.getBufferManager().fetchPage(new DummyLockContext(), pnum, false); + this.pinnedPages.put(pnum, page); + } + + private void unpinPage(int partNum, int pageNum) { + long pnum = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + this.pinnedPages.remove(pnum).unpin(); + } + + private void evictPage(int partNum, int pageNum) { + long pnum = DiskSpaceManager.getVirtualPageNum(partNum, pageNum); + this.d.getBufferManager().evict(pnum); + numIOs = d.getBufferManager().getNumIOs(); + } + + private void pinMetadata() { + // hard-coded mess, but works as long as the first two tables created are the source operators + pinPage(1, 0); // information_schema.tables header page + pinPage(1, 3); // information_schema.tables entry for source + } + + @Test + @Category(PublicTests.class) + public void testSortRun() { + try(Transaction transaction = d.beginTransaction()) { + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "table"); + List records = new ArrayList<>(); + List recordsToShuffle = new ArrayList<>(); + for (int i = 0; i < 400 * 3; i++) { + Record r = TestUtils.createRecordWithAllTypesWithValue(i); + records.add(r); + recordsToShuffle.add(r); + } + Collections.shuffle(recordsToShuffle, new Random(42)); + + pinMetadata(); + startCountIOs(); + + SortOperator s = new SortOperator(transaction.getTransactionContext(), "table", + new SortRecordComparator(1)); + checkIOs(0); + + SortOperator.Run r = s.createRun(); + checkIOs(NEW_TABLE_IOS); // information_schema.tables row + header page of table + + r.addRecords(recordsToShuffle); + checkIOs(3); + + startCountIOs(); + SortOperator.Run sortedRun = s.sortRun(r); + checkIOs((3 + FIRST_ACCESS_IOS) + (3 + NEW_TABLE_IOS)); + + Iterator iter = sortedRun.iterator(); + int i = 0; + while (iter.hasNext() && i < 400 * 3) { + assertEquals("mismatch at record " + i, records.get(i), iter.next()); + i++; + } + assertFalse("too many records", iter.hasNext()); + assertEquals("too few records", 400 * 3, i); + } + } + + @Test + @Category(PublicTests.class) + public void testMergeSortedRuns() { + try(Transaction transaction = d.beginTransaction()) { + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "table"); + List records = new ArrayList<>(); + + pinMetadata(); + startCountIOs(); + + SortOperator s = new SortOperator(transaction.getTransactionContext(), "table", + new SortRecordComparator(1)); + checkIOs(0); + SortOperator.Run r1 = s.createRun(); + SortOperator.Run r2 = s.createRun(); + checkIOs(2 * NEW_TABLE_IOS); + + for (int i = 0; i < 400 * 3; i++) { + Record r = TestUtils.createRecordWithAllTypesWithValue(i); + records.add(r); + if (i % 2 == 0) { + r1.addRecord(r.getValues()); + } else { + r2.addRecord(r.getValues()); + } + } + List runs = new ArrayList<>(); + runs.add(r1); + runs.add(r2); + checkIOs(2 * 2); + + startCountIOs(); + SortOperator.Run mergedSortedRuns = s.mergeSortedRuns(runs); + checkIOs((2 * (2 + FIRST_ACCESS_IOS)) + (3 + NEW_TABLE_IOS)); + + Iterator iter = mergedSortedRuns.iterator(); + int i = 0; + while (iter.hasNext() && i < 400 * 3) { + assertEquals("mismatch at record " + i, records.get(i), iter.next()); + i++; + } + assertFalse("too many records", iter.hasNext()); + assertEquals("too few records", 400 * 3, i); + checkIOs(0); + } + } + + @Test + @Category(PublicTests.class) + public void testMergePass() { + try(Transaction transaction = d.beginTransaction()) { + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "table"); + List records1 = new ArrayList<>(); + List records2 = new ArrayList<>(); + + pinMetadata(); + startCountIOs(); + + SortOperator s = new SortOperator(transaction.getTransactionContext(), "table", + new SortRecordComparator(1)); + checkIOs(0); + + SortOperator.Run r1 = s.createRun(); + SortOperator.Run r2 = s.createRun(); + SortOperator.Run r3 = s.createRun(); + SortOperator.Run r4 = s.createRun(); + checkIOs(4 * NEW_TABLE_IOS); + + for (int i = 0; i < 400 * 4; i++) { + Record r = TestUtils.createRecordWithAllTypesWithValue(i); + if (i % 4 == 0) { + r1.addRecord(r.getValues()); + records2.add(r); + } else if (i % 4 == 1) { + r2.addRecord(r.getValues()); + records1.add(r); + } else if (i % 4 == 2) { + r3.addRecord(r.getValues()); + records1.add(r); + } else { + r4.addRecord(r.getValues()); + records2.add(r); + } + } + checkIOs(4 * 1); + + List runs = new ArrayList<>(); + runs.add(r3); + runs.add(r2); + runs.add(r1); + runs.add(r4); + + startCountIOs(); + List result = s.mergePass(runs); + assertEquals("wrong number of runs", 2, result.size()); + checkIOs((4 * (1 + FIRST_ACCESS_IOS)) + (2 * (2 + NEW_TABLE_IOS))); + + Iterator iter1 = result.get(0).iterator(); + Iterator iter2 = result.get(1).iterator(); + int i = 0; + while (iter1.hasNext() && i < 400 * 2) { + assertEquals("mismatch at record " + i, records1.get(i), iter1.next()); + i++; + } + assertFalse("too many records", iter1.hasNext()); + assertEquals("too few records", 400 * 2, i); + i = 0; + while (iter2.hasNext() && i < 400 * 2) { + assertEquals("mismatch at record " + i, records2.get(i), iter2.next()); + i++; + } + assertFalse("too many records", iter2.hasNext()); + assertEquals("too few records", 400 * 2, i); + checkIOs(0); + } + } + + @Test + @Category(PublicTests.class) + public void testSortNoChange() { + try(Transaction transaction = d.beginTransaction()) { + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "table"); + Record[] records = new Record[400 * 3]; + for (int i = 0; i < 400 * 3; i++) { + Record r = TestUtils.createRecordWithAllTypesWithValue(i); + records[i] = r; + transaction.getTransactionContext().addRecord("table", r.getValues()); + } + + pinMetadata(); + startCountIOs(); + + SortOperator s = new SortOperator(transaction.getTransactionContext(), "table", + new SortRecordComparator(1)); + checkIOs(0); + + String sortedTableName = s.sort(); + checkIOs((3 + FIRST_ACCESS_IOS) + (3 + NEW_TABLE_IOS)); + + Iterator iter = transaction.getTransactionContext().getRecordIterator(sortedTableName); + int i = 0; + while (iter.hasNext() && i < 400 * 3) { + assertEquals("mismatch at record " + i, records[i], iter.next()); + i++; + } + assertFalse("too many records", iter.hasNext()); + assertEquals("too few records", 400 * 3, i); + checkIOs(0); + } + } + + @Test + @Category(PublicTests.class) + public void testSortBackwards() { + try(Transaction transaction = d.beginTransaction()) { + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "table"); + Record[] records = new Record[400 * 3]; + for (int i = 400 * 3; i > 0; i--) { + Record r = TestUtils.createRecordWithAllTypesWithValue(i); + records[i - 1] = r; + transaction.getTransactionContext().addRecord("table", r.getValues()); + } + + pinMetadata(); + startCountIOs(); + + SortOperator s = new SortOperator(transaction.getTransactionContext(), "table", + new SortRecordComparator(1)); + checkIOs(0); + + String sortedTableName = s.sort(); + checkIOs((3 + FIRST_ACCESS_IOS) + (3 + NEW_TABLE_IOS)); + + Iterator iter = transaction.getTransactionContext().getRecordIterator(sortedTableName); + int i = 0; + while (iter.hasNext() && i < 400 * 3) { + assertEquals("mismatch at record " + i, records[i], iter.next()); + i++; + } + assertFalse("too many records", iter.hasNext()); + assertEquals("too few records", 400 * 3, i); + checkIOs(0); + } + } + + @Test + @Category(PublicTests.class) + public void testSortRandomOrder() { + try(Transaction transaction = d.beginTransaction()) { + transaction.createTable(TestUtils.createSchemaWithAllTypes(), "table"); + List records = new ArrayList<>(); + List recordsToShuffle = new ArrayList<>(); + for (int i = 0; i < 400 * 3; i++) { + Record r = TestUtils.createRecordWithAllTypesWithValue(i); + records.add(r); + recordsToShuffle.add(r); + } + Collections.shuffle(recordsToShuffle, new Random(42)); + for (Record r : recordsToShuffle) { + transaction.getTransactionContext().addRecord("table", r.getValues()); + } + + pinMetadata(); + startCountIOs(); + + SortOperator s = new SortOperator(transaction.getTransactionContext(), "table", + new SortRecordComparator(1)); + checkIOs(0); + + String sortedTableName = s.sort(); + checkIOs((3 + FIRST_ACCESS_IOS) + (3 + NEW_TABLE_IOS)); + + Iterator iter = transaction.getTransactionContext().getRecordIterator(sortedTableName); + int i = 0; + while (iter.hasNext() && i < 400 * 3) { + assertEquals("mismatch at record " + i, records.get(i), iter.next()); + i++; + } + assertFalse("too many records", iter.hasNext()); + assertEquals("too few records", 400 * 3, i); + checkIOs(0); + } + } + +} diff --git a/src/test/java/edu/berkeley/cs186/database/query/TestSourceOperator.java b/src/test/java/edu/berkeley/cs186/database/query/TestSourceOperator.java new file mode 100644 index 0000000..57cdb9a --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/query/TestSourceOperator.java @@ -0,0 +1,93 @@ +package edu.berkeley.cs186.database.query; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import edu.berkeley.cs186.database.TestUtils; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.Table; +import edu.berkeley.cs186.database.table.stats.TableStats; + +public class TestSourceOperator extends QueryOperator { + private List recordList; + private Schema setSchema; + private int numRecords; + + public TestSourceOperator() { + super(OperatorType.SEQSCAN, null); + this.recordList = null; + this.setSchema = null; + this.numRecords = 100; + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + public TestSourceOperator(List recordIterator, Schema schema) { + super(OperatorType.SEQSCAN); + + this.recordList = recordIterator; + this.setOutputSchema(schema); + this.setSchema = schema; + this.numRecords = 100; + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public boolean isSequentialScan() { + return false; + } + + public TestSourceOperator(int numRecords) { + super(OperatorType.SEQSCAN, null); + this.recordList = null; + this.setSchema = null; + this.numRecords = numRecords; + + this.stats = this.estimateStats(); + this.cost = this.estimateIOCost(); + } + + @Override + public Iterator execute() { + if (this.recordList == null) { + ArrayList recordList = new ArrayList(); + for (int i = 0; i < this.numRecords; i++) { + recordList.add(TestUtils.createRecordWithAllTypes()); + } + + return recordList.iterator(); + } + return this.recordList.iterator(); + } + + @Override + public Iterator iterator() { + return this.execute(); + } + + @Override + protected Schema computeSchema() { + if (this.setSchema == null) { + return TestUtils.createSchemaWithAllTypes(); + } + return this.setSchema; + } + + @Override + public TableStats estimateStats() { + Schema schema = this.computeSchema(); + return new TableStats(schema, Table.computeNumRecordsPerPage(BufferManager.EFFECTIVE_PAGE_SIZE, + schema)); + } + + @Override + public int estimateIOCost() { + return 1; + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/recovery/ARIESRecoveryManagerNoLocking.java b/src/test/java/edu/berkeley/cs186/database/recovery/ARIESRecoveryManagerNoLocking.java new file mode 100644 index 0000000..3ecf281 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/recovery/ARIESRecoveryManagerNoLocking.java @@ -0,0 +1,26 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.concurrency.LockContext; + +import java.util.function.Function; + +// Instrumented version of ARIESRecoveryManager for testing, without locking so that tests pass without HW4. +class ARIESRecoveryManagerNoLocking extends ARIESRecoveryManager { + long transactionCounter = 0L; + + ARIESRecoveryManagerNoLocking(LockContext dbContext, + Function newTransaction) { + super(dbContext, newTransaction, null, null, true); + super.updateTransactionCounter = x -> transactionCounter = x; + super.getTransactionCounter = () -> transactionCounter; + } + + @Override + public long logFreePage(long transNum, long pageNum) { + long rv = super.logFreePage(transNum, pageNum); + transactionTable.get(transNum).touchedPages.add(pageNum); + dirtyPageTable.remove(pageNum); + return rv; + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/recovery/DummyTransaction.java b/src/test/java/edu/berkeley/cs186/database/recovery/DummyTransaction.java new file mode 100644 index 0000000..4b571ec --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/recovery/DummyTransaction.java @@ -0,0 +1,318 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.AbstractTransaction; +import edu.berkeley.cs186.database.AbstractTransactionContext; +import edu.berkeley.cs186.database.TransactionContext; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.index.BPlusTreeMetadata; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.query.QueryPlan; +import edu.berkeley.cs186.database.table.Record; +import edu.berkeley.cs186.database.table.RecordId; +import edu.berkeley.cs186.database.table.Schema; +import edu.berkeley.cs186.database.table.Table; +import edu.berkeley.cs186.database.table.stats.TableStats; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.function.UnaryOperator; + +/** + * Dummy transaction that does nothing except maintain transaction + * number and status. + */ +class DummyTransaction extends AbstractTransaction { + private long transNum; + boolean cleanedUp = false; + + private static Map transactions = new HashMap<>(); + + private DummyTransaction(long transNum) { + this.transNum = transNum; + } + + static DummyTransaction create(long transNum) { + if (!transactions.containsKey(transNum)) { + transactions.put(transNum, new DummyTransaction(transNum)); + } + return transactions.get(transNum); + } + + static void cleanupTransactions() { + for (DummyTransaction transaction : transactions.values()) { + transaction.cleanedUp = false; + transaction.setStatus(Status.RUNNING); + } + } + + @Override + protected void startCommit() {} + + @Override + protected void startRollback() {} + + @Override + public void cleanup() { + assert (!cleanedUp && getStatus() != Status.COMPLETE); + cleanedUp = true; + } + + @Override + public long getTransNum() { + return transNum; + } + + @Override + public void createTable(Schema s, String tableName) {} + + @Override + public void dropTable(String tableName) {} + + @Override + public void dropAllTables() {} + + @Override + public void createIndex(String tableName, String columnName, boolean bulkLoad) {} + + @Override + public void dropIndex(String tableName, String columnName) {} + + @Override + public QueryPlan query(String tableName) { + return null; + } + + @Override + public QueryPlan query(String tableName, String alias) { + return null; + } + + @Override + public void insert(String tableName, List values) {} + + @Override + public void update(String tableName, String targetColumnName, UnaryOperator targetValue) {} + + @Override + public void update(String tableName, String targetColumnName, UnaryOperator targetValue, + String predColumnName, PredicateOperator predOperator, DataBox predValue) {} + + @Override + public void delete(String tableName, String predColumnName, PredicateOperator predOperator, + DataBox predValue) {} + + @Override + public void savepoint(String savepointName) {} + + @Override + public void rollbackToSavepoint(String savepointName) {} + + @Override + public void releaseSavepoint(String savepointName) {} + + @Override + public Schema getSchema(String tableName) { + return null; + } + + @Override + public Schema getFullyQualifiedSchema(String tableName) { + return null; + } + + @Override + public TableStats getStats(String tableName) { + return null; + } + + @Override + public int getNumDataPages(String tableName) { + return 0; + } + + @Override + public int getNumEntriesPerPage(String tableName) { + return 0; + } + + @Override + public int getEntrySize(String tableName) { + return 0; + } + + @Override + public long getNumRecords(String tableName) { + return 0; + } + + @Override + public int getTreeOrder(String tableName, String columnName) { + return 0; + } + + @Override + public int getTreeHeight(String tableName, String columnName) { + return 0; + } + + @Override + public TransactionContext getTransactionContext() { + return new DummyTransactionContext(); + } + + private class DummyTransactionContext extends AbstractTransactionContext { + @Override + public long getTransNum() { + return transNum; + } + + @Override + public int getWorkMemSize() { + return 0; + } + + @Override + public void close() {} + + @Override + public String createTempTable(Schema schema) { + return null; + } + + @Override + public void deleteAllTempTables() {} + + @Override + public void setAliasMap(Map aliasMap) {} + + @Override + public void clearAliasMap() {} + + @Override + public boolean indexExists(String tableName, String columnName) { + return false; + } + + @Override + public void updateIndexMetadata(BPlusTreeMetadata metadata) {} + + @Override + public Iterator sortedScan(String tableName, String columnName) { + return null; + } + + @Override + public Iterator sortedScanFrom(String tableName, String columnName, DataBox startValue) { + return null; + } + + @Override + public Iterator lookupKey(String tableName, String columnName, DataBox key) { + return null; + } + + @Override + public BacktrackingIterator getRecordIterator(String tableName) { + return null; + } + + @Override + public BacktrackingIterator getPageIterator(String tableName) { + return null; + } + + @Override + public BacktrackingIterator getBlockIterator(String tableName, Iterator block, + int maxPages) { + return null; + } + + @Override + public boolean contains(String tableName, String columnName, DataBox key) { + return false; + } + + @Override + public RecordId addRecord(String tableName, List values) { + return null; + } + + @Override + public RecordId deleteRecord(String tableName, RecordId rid) { + return null; + } + + @Override + public Record getRecord(String tableName, RecordId rid) { + return null; + } + + @Override + public RecordId updateRecord(String tableName, List values, RecordId rid) { + return null; + } + + @Override + public void runUpdateRecordWhere(String tableName, String targetColumnName, + UnaryOperator targetValue, String predColumnName, PredicateOperator predOperator, + DataBox predValue) {} + + @Override + public void runDeleteRecordWhere(String tableName, String predColumnName, + PredicateOperator predOperator, DataBox predValue) {} + + @Override + public Schema getSchema(String tableName) { + return null; + } + + @Override + public Schema getFullyQualifiedSchema(String tableName) { + return null; + } + + @Override + public Table getTable(String tableName) { + return null; + } + + @Override + public TableStats getStats(String tableName) { + return null; + } + + @Override + public int getNumDataPages(String tableName) { + return 0; + } + + @Override + public int getNumEntriesPerPage(String tableName) { + return 0; + } + + @Override + public int getEntrySize(String tableName) { + return 0; + } + + @Override + public long getNumRecords(String tableName) { + return 0; + } + + @Override + public int getTreeOrder(String tableName, String columnName) { + return 0; + } + + @Override + public int getTreeHeight(String tableName, String columnName) { + return 0; + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudent.java b/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudent.java new file mode 100644 index 0000000..fa47186 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudent.java @@ -0,0 +1,229 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.categories.HW5Tests; +import edu.berkeley.cs186.database.categories.StudentTests; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.DiskSpaceManagerImpl; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.LRUEvictionPolicy; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import java.util.*; +import java.util.function.Consumer; + +import static junit.framework.TestCase.assertTrue; +import static junit.framework.TestCase.fail; +import static org.junit.Assert.*; +import static org.junit.Assert.assertNotNull; + +/** + * File for student tests for HW5 (Recovery). Tests are run through + * TestARIESStudentRunner for grading purposes. + */ +@Category({HW5Tests.class, StudentTests.class}) +public class TestARIESStudent { + private String testDir; + private RecoveryManager recoveryManager; + private final Queue> redoMethods = new ArrayDeque<>(); + + // 1 second per test + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setup() throws Exception { + testDir = tempFolder.newFolder("test-dir").getAbsolutePath(); + recoveryManager = loadRecoveryManager(testDir); + DummyTransaction.cleanupTransactions(); + LogRecord.onRedoHandler(t -> {}); + } + + @After + public void cleanup() throws Exception {} + + @Test + public void testStudentAnalysis() throws Exception { + // TODO(hw5): write your own test on restartAnalysis only + // You should use loadRecoveryManager instead of new ARIESRecoveryManager(..) to + // create the recovery manager, and use runAnalysis(inner) instead of + // inner.restartAnalysis() to call the analysis routine. + } + + @Test + public void testStudentRedo() throws Exception { + // TODO(hw5): write your own test on restartRedo only + // You should use loadRecoveryManager instead of new ARIESRecoveryManager(..) to + // create the recovery manager, and use runRedo(inner) instead of + // inner.restartRedo() to call the analysis routine. + } + + @Test + public void testStudentUndo() throws Exception { + // TODO(hw5): write your own test on restartUndo only + // You should use loadRecoveryManager instead of new ARIESRecoveryManager(..) to + // create the recovery manager, and use runUndo(inner) instead of + // inner.restartUndo() to call the analysis routine. + } + + @Test + public void testStudentIntegration() throws Exception { + // TODO(hw5): write your own test on all of RecoveryManager + // You should use loadRecoveryManager instead of new ARIESRecoveryManager(..) to + // create the recovery manager. + } + + // TODO(hw5): add as many (ungraded) tests as you want for testing! + + @Test + public void testCase() throws Exception { + // TODO(hw5): write your own test! (ungraded) + } + + @Test + public void anotherTestCase() throws Exception { + // TODO(hw5): write your own test!!! (ungraded) + } + + @Test + public void yetAnotherTestCase() throws Exception { + // TODO(hw5): write your own test!!!!! (ungraded) + } + + /************************************************************************* + * Helpers for writing tests. * + * Do not change the signature of any of the following methods. * + *************************************************************************/ + + /** + * Helper to set up checks for redo. The first call to LogRecord.redo will + * call the first method in METHODS, the second call to the second method in METHODS, + * and so on. Call this method before the redo pass, and call finishRedoChecks + * after the redo pass. + */ + private void setupRedoChecks(Collection> methods) { + for (final Consumer method : methods) { + redoMethods.add(record -> { + method.accept(record); + LogRecord.onRedoHandler(redoMethods.poll()); + }); + } + redoMethods.add(record -> { + fail("LogRecord#redo() called too many times"); + }); + LogRecord.onRedoHandler(redoMethods.poll()); + } + + /** + * Helper to finish checks for redo. Call this after the redo pass (or undo pass)- + * if not enough redo calls were performed, an error is thrown. + * + * If setupRedoChecks is used for the redo pass, and this method is not called before + * the undo pass, and the undo pass calls undo at least once, an error may be incorrectly thrown. + */ + private void finishRedoChecks() { + assertTrue("LogRecord#redo() not called enough times", redoMethods.isEmpty()); + LogRecord.onRedoHandler(record -> {}); + } + + /** + * Loads the recovery manager from disk. + * @param dir testDir + * @return recovery manager, loaded from disk + */ + protected RecoveryManager loadRecoveryManager(String dir) throws Exception { + RecoveryManager recoveryManager = new ARIESRecoveryManagerNoLocking( + new DummyLockContext(new Pair<>("database", 0L)), + DummyTransaction::create + ); + DiskSpaceManager diskSpaceManager = new DiskSpaceManagerImpl(dir, recoveryManager); + BufferManager bufferManager = new BufferManagerImpl(diskSpaceManager, recoveryManager, 32, + new LRUEvictionPolicy()); + boolean isLoaded = true; + try { + diskSpaceManager.allocPart(0); + diskSpaceManager.allocPart(1); + for (int i = 0; i < 10; ++i) { + diskSpaceManager.allocPage(DiskSpaceManager.getVirtualPageNum(1, i)); + } + isLoaded = false; + } catch (IllegalStateException e) { + // already loaded + } + recoveryManager.setManagers(diskSpaceManager, bufferManager); + if (!isLoaded) { + recoveryManager.initialize(); + } + return recoveryManager; + } + + /** + * Flushes everything to disk, but does not call RecoveryManager#shutdown. Similar + * to pulling the plug on the database at a time when no changes are in memory. You + * can simulate a shutdown where certain changes _are_ in memory, by simply never + * applying them (i.e. write a log record, but do not make the changes on the + * buffer manager/disk space manager). + */ + protected void shutdownRecoveryManager(RecoveryManager recoveryManager) throws Exception { + ARIESRecoveryManager arm = (ARIESRecoveryManager) recoveryManager; + arm.logManager.close(); + arm.bufferManager.evictAll(); + arm.bufferManager.close(); + arm.diskSpaceManager.close(); + DummyTransaction.cleanupTransactions(); + } + + protected BufferManager getBufferManager(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).bufferManager; + } + + protected DiskSpaceManager getDiskSpaceManager(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).diskSpaceManager; + } + + protected LogManager getLogManager(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).logManager; + } + + protected List getLockRequests(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).lockRequests; + } + + protected long getTransactionCounter(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManagerNoLocking) recoveryManager).transactionCounter; + } + + protected Map getDirtyPageTable(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).dirtyPageTable; + } + + protected Map getTransactionTable(RecoveryManager recoveryManager) + throws Exception { + return ((ARIESRecoveryManager) recoveryManager).transactionTable; + } + + protected void runAnalysis(RecoveryManager recoveryManager) throws Exception { + ((ARIESRecoveryManager) recoveryManager).restartAnalysis(); + } + + protected void runRedo(RecoveryManager recoveryManager) throws Exception { + ((ARIESRecoveryManager) recoveryManager).restartRedo(); + } + + protected void runUndo(RecoveryManager recoveryManager) throws Exception { + ((ARIESRecoveryManager) recoveryManager).restartUndo(); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudentRunner.java b/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudentRunner.java new file mode 100644 index 0000000..058d98e --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudentRunner.java @@ -0,0 +1,3 @@ +package edu.berkeley.cs186.database.recovery; + +public class TestARIESStudentRunner extends TestARIESStudentRunnerBase { @Override String getSuffix() { return ""; } } diff --git a/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudentRunnerBase.java b/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudentRunnerBase.java new file mode 100644 index 0000000..cd703fb --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudentRunnerBase.java @@ -0,0 +1,559 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.categories.HW5Tests; +import edu.berkeley.cs186.database.categories.StudentTestRunner; +import edu.berkeley.cs186.database.categories.StudentTests; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.DiskSpaceManagerImpl; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.LRUEvictionPolicy; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +import static junit.framework.TestCase.assertNotNull; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNull; + +@Category({HW5Tests.class, StudentTestRunner.class}) +public abstract class TestARIESStudentRunnerBase { + private Class recoveryManagerClass; + + // 1 second per test + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + abstract String getSuffix(); + + @Before + @SuppressWarnings("unchecked") + public void setup() throws Exception { + assertNotNull("TestARIESStudent should have @Category({HW5Tests.class, StudentTests.class})", + TestARIESStudent.class.getAnnotation(Category.class)); + assertArrayEquals("TestARIESStudent should have @Category({HW5Tests.class, StudentTests.class})", + new Class[] { HW5Tests.class, StudentTests.class }, TestARIESStudent.class.getAnnotation( + Category.class).value()); + for (Method m : TestARIESStudent.class.getMethods()) { + assertNull("TestARIESStudent methods should not have @Category", m.getAnnotation(Category.class)); + } + + String className = ARIESRecoveryManager.class.getCanonicalName() + getSuffix(); + recoveryManagerClass = (Class) Class.forName(className); + + DummyTransaction.cleanupTransactions(); + LogRecord.onRedoHandler(t -> {}); + } + + private static class AnalysisInstance extends TestInstance { + private AnalysisInstance(Class recoveryManagerClass) throws Exception { + super(recoveryManagerClass, true); + } + + @Override + protected void runRedo(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentAnalysis"); + } + + @Override + protected void runUndo(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentAnalysis"); + } + } + + @Test + public void testStudentAnalysis() throws Exception { + TestInstance ti = new AnalysisInstance(recoveryManagerClass); + try { + ti.tempFolder.create(); + try { + ti.setup(); + ti.testStudentAnalysis(); + } finally { + ti.cleanup(); + } + } finally { + ti.tempFolder.delete(); + } + } + + private static class RedoInstance extends TestInstance { + private RedoInstance(Class recoveryManagerClass) throws Exception { + super(recoveryManagerClass, true); + } + + @Override + protected void runAnalysis(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentRedo"); + } + + @Override + protected void runUndo(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentRedo"); + } + } + + @Test + public void testStudentRedo() throws Exception { + TestInstance ti = new RedoInstance(recoveryManagerClass); + try { + ti.tempFolder.create(); + try { + ti.setup(); + ti.testStudentRedo(); + } finally { + ti.cleanup(); + } + } finally { + ti.tempFolder.delete(); + } + } + + private static class UndoInstance extends TestInstance { + private UndoInstance(Class recoveryManagerClass) throws Exception { + super(recoveryManagerClass, true); + } + + @Override + protected void runAnalysis(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentUndo"); + } + + @Override + protected void runRedo(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentUndo"); + } + } + + @Test + public void testStudentUndo() throws Exception { + TestInstance ti = new UndoInstance(recoveryManagerClass); + try { + ti.tempFolder.create(); + try { + ti.setup(); + ti.testStudentUndo(); + } finally { + ti.cleanup(); + } + } finally { + ti.tempFolder.delete(); + } + } + + private static class IntegrationInstance extends TestInstance { + private IntegrationInstance(Class recoveryManagerClass) throws + Exception { + super(recoveryManagerClass, false); + } + + @Override + protected void runAnalysis(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentIntegration"); + } + + @Override + protected void runRedo(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentIntegration"); + } + + @Override + protected void runUndo(RecoveryManager recoveryManager) { + throw new UnsupportedOperationException("not allowed in testStudentIntegration"); + } + } + + @Test + public void testStudentIntegration() throws Exception { + TestInstance ti = new IntegrationInstance(recoveryManagerClass); + try { + ti.tempFolder.create(); + try { + ti.setup(); + ti.testStudentIntegration(); + } finally { + ti.cleanup(); + } + } finally { + ti.tempFolder.delete(); + } + } + + private static abstract class TestInstance extends TestARIESStudent { + private final Class recoveryManagerClass; + private final Constructor recoveryManagerConstructor; + private final boolean limited; + + private TestInstance(Class recoveryManagerClass, + boolean limited) throws Exception { + this.recoveryManagerClass = recoveryManagerClass; + this.recoveryManagerConstructor = recoveryManagerClass.getDeclaredConstructor( + LockContext.class, + Function.class, + Consumer.class, + Supplier.class, + boolean.class + ); + this.limited = limited; + } + + @Override + protected RecoveryManager loadRecoveryManager(String dir) throws Exception { + RecoveryManager recoveryManager = new DelegatedRecoveryManager(recoveryManagerClass, + recoveryManagerConstructor); + DiskSpaceManager diskSpaceManager = new DiskSpaceManagerImpl(dir, recoveryManager); + BufferManager bufferManager = new BufferManagerImpl(diskSpaceManager, recoveryManager, 32, + new LRUEvictionPolicy()); + boolean isLoaded = true; + try { + diskSpaceManager.allocPart(0); + diskSpaceManager.allocPart(1); + for (int i = 0; i < 10; ++i) { + diskSpaceManager.allocPage(DiskSpaceManager.getVirtualPageNum(1, i)); + } + isLoaded = false; + } catch (IllegalStateException e) { + // already loaded + } + recoveryManager.setManagers(diskSpaceManager, bufferManager); + if (!isLoaded) { + recoveryManager.initialize(); + } + if (limited) { + recoveryManager = new LimitedRecoveryManager((DelegatedRecoveryManager) recoveryManager); + } + return recoveryManager; + } + + @Override + protected void shutdownRecoveryManager(RecoveryManager recoveryManager) throws Exception { + getLogManager(recoveryManager).close(); + getBufferManager(recoveryManager).evictAll(); + getBufferManager(recoveryManager).close(); + getDiskSpaceManager(recoveryManager).close(); + DummyTransaction.cleanupTransactions(); + } + + @Override + protected BufferManager getBufferManager(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + return (BufferManager) recoveryManagerClass.getDeclaredField("bufferManager").get(drm.inner); + } + + @Override + protected DiskSpaceManager getDiskSpaceManager(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + return (DiskSpaceManager) recoveryManagerClass.getDeclaredField("diskSpaceManager").get(drm.inner); + } + + @Override + protected LogManager getLogManager(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + return (LogManager) recoveryManagerClass.getDeclaredField("logManager").get(drm.inner); + } + + @Override + @SuppressWarnings("unchecked") + protected List getLockRequests(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + return (List) recoveryManagerClass.getDeclaredField("lockRequests").get(drm.inner); + } + + @Override + protected long getTransactionCounter(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + return drm.transactionCounter; + } + + @Override + @SuppressWarnings("unchecked") + protected Map getDirtyPageTable(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + return (Map) recoveryManagerClass.getDeclaredField("dirtyPageTable").get(drm.inner); + } + + @Override + @SuppressWarnings("unchecked") + protected Map getTransactionTable(RecoveryManager recoveryManager) + throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + return (Map) + recoveryManagerClass.getDeclaredField("transactionTable").get(drm.inner); + } + + @Override + protected void runAnalysis(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + recoveryManagerClass.getDeclaredMethod("restartAnalysis").invoke(drm.inner); + } + + @Override + protected void runRedo(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + recoveryManagerClass.getDeclaredMethod("restartRedo").invoke(drm.inner); + } + + @Override + protected void runUndo(RecoveryManager recoveryManager) throws Exception { + DelegatedRecoveryManager drm = limited ? ((LimitedRecoveryManager) recoveryManager).inner : (( + DelegatedRecoveryManager) recoveryManager); + recoveryManagerClass.getDeclaredMethod("restartUndo").invoke(drm.inner); + } + } + + private static class LimitedRecoveryManager implements RecoveryManager { + private DelegatedRecoveryManager inner; + + private LimitedRecoveryManager(DelegatedRecoveryManager inner) { + this.inner = inner; + } + + @Override + public void initialize() { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void setManagers(DiskSpaceManager diskSpaceManager, BufferManager bufferManager) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void startTransaction(Transaction transaction) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public long commit(long transNum) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public long abort(long transNum) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public long end(long transNum) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void pageFlushHook(long pageLSN) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void diskIOHook(long pageNum) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public long logPageWrite(long transNum, long pageNum, short pageOffset, byte[] before, + byte[] after) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public long logAllocPart(long transNum, int partNum) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public long logFreePart(long transNum, int partNum) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public long logAllocPage(long transNum, long pageNum) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public long logFreePage(long transNum, long pageNum) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void savepoint(long transNum, String name) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void releaseSavepoint(long transNum, String name) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void rollbackToSavepoint(long transNum, String name) { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void checkpoint() { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public Runnable restart() { + throw new UnsupportedOperationException("this method may not be used"); + } + + @Override + public void close() { + throw new UnsupportedOperationException("this method may not be used"); + } + } + + private static class DelegatedRecoveryManager implements RecoveryManager { + private final Class recoveryManagerClass; + RecoveryManager inner; + long transactionCounter = 0L; + + DelegatedRecoveryManager(Class recoveryManagerClass, + Constructor recoveryManagerConstructor) throws Exception { + this.recoveryManagerClass = recoveryManagerClass; + this.inner = recoveryManagerConstructor.newInstance( + new DummyLockContext(new Pair<>("database", 0L)), + (Function) DummyTransaction::create, + (Consumer) t -> transactionCounter = t, + (Supplier) () -> transactionCounter, + true + ); + } + + @Override + public void initialize() { + inner.initialize(); + } + + @Override + public void setManagers(DiskSpaceManager diskSpaceManager, BufferManager bufferManager) { + inner.setManagers(diskSpaceManager, bufferManager); + } + + @Override + public void startTransaction(Transaction transaction) { + inner.startTransaction(transaction); + } + + @Override + public long commit(long transNum) { + return inner.commit(transNum); + } + + @Override + public long abort(long transNum) { + return inner.abort(transNum); + } + + @Override + public long end(long transNum) { + return inner.end(transNum); + } + + @Override + public void pageFlushHook(long pageLSN) { + inner.pageFlushHook(pageLSN); + } + + @Override + public void diskIOHook(long pageNum) { + inner.diskIOHook(pageNum); + } + + @Override + public long logPageWrite(long transNum, long pageNum, short pageOffset, byte[] before, + byte[] after) { + return inner.logPageWrite(transNum, pageNum, pageOffset, before, after); + } + + @Override + public long logAllocPart(long transNum, int partNum) { + return inner.logAllocPart(transNum, partNum); + } + + @Override + public long logFreePart(long transNum, int partNum) { + return inner.logFreePart(transNum, partNum); + } + + @Override + public long logAllocPage(long transNum, long pageNum) { + return inner.logAllocPage(transNum, pageNum); + } + + @Override + public long logFreePage(long transNum, long pageNum) { + long rv = inner.logFreePage(transNum, pageNum); + try { + Map transactionTable = (Map) + recoveryManagerClass.getDeclaredField("transactionTable").get(inner); + Map dirtyPageTable = (Map) + recoveryManagerClass.getDeclaredField("dirtyPageTable").get(inner); + transactionTable.get(transNum).touchedPages.add(pageNum); + dirtyPageTable.remove(pageNum); + } catch (Exception e) { + throw new RuntimeException(e); + } + return rv; + } + + @Override + public void savepoint(long transNum, String name) { + inner.savepoint(transNum, name); + } + + @Override + public void releaseSavepoint(long transNum, String name) { + inner.releaseSavepoint(transNum, name); + } + + @Override + public void rollbackToSavepoint(long transNum, String name) { + inner.rollbackToSavepoint(transNum, name); + } + + @Override + public void checkpoint() { + inner.checkpoint(); + } + + @Override + public Runnable restart() { + return inner.restart(); + } + + @Override + public void close() { + inner.close(); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/recovery/TestLogManager.java b/src/test/java/edu/berkeley/cs186/database/recovery/TestLogManager.java new file mode 100644 index 0000000..60030f4 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/recovery/TestLogManager.java @@ -0,0 +1,133 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.categories.SystemTests; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.concurrency.LockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.MemoryDiskSpaceManager; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.ClockEvictionPolicy; +import edu.berkeley.cs186.database.memory.Page; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +@Category(SystemTests.class) +public class TestLogManager { + private LogManager logManager; + private BufferManager bufferManager; + + @Before + public void setup() { + DiskSpaceManager diskSpaceManager = new MemoryDiskSpaceManager(); + diskSpaceManager.allocPart(0); + this.bufferManager = new BufferManagerImpl(diskSpaceManager, new DummyRecoveryManager(), 1024, + new ClockEvictionPolicy()); + logManager = new LogManagerImpl(bufferManager); + } + + @After + public void cleanup() { + logManager.close(); + bufferManager.close(); + } + + @Test + public void testAppendFetch() { + LogRecord expected = new MasterLogRecord(1234); + + logManager.appendToLog(expected); + LogRecord record = logManager.fetchLogRecord(0); + + assertEquals(expected, record); + } + + @Test + public void testAppendScan() { + LogRecord expected = new MasterLogRecord(1234); + + logManager.appendToLog(expected); + LogRecord record = logManager.scanFrom(0).next(); + + assertEquals(expected, record); + } + + @Test + public void testAppendIterator() { + LogRecord expected = new MasterLogRecord(1234); + + logManager.appendToLog(expected); + LogRecord record = logManager.iterator().next(); + + assertEquals(expected, record); + } + + @Test + public void testFlushedLSN() { + logManager.appendToLog(new MasterLogRecord(1234)); + logManager.flushToLSN(9999); + + assertEquals(9999, logManager.getFlushedLSN()); + } + + @Test + public void testMultiPageScan() { + for (int i = 0; i < 10000; ++i) { + logManager.appendToLog(new MasterLogRecord(i)); + } + + Iterator iter = logManager.scanFrom(90000); + for (int i = 9 * (DiskSpaceManager.PAGE_SIZE / 9); i < 10000; ++i) { + assertEquals(new MasterLogRecord(i), iter.next()); + } + assertFalse(iter.hasNext()); + } + + @Test + public void testRewriteMasterRecord() { + for (int i = 0; i < 1000; ++i) { + logManager.appendToLog(new MasterLogRecord(i)); + } + logManager.rewriteMasterRecord(new MasterLogRecord(77)); + logManager.rewriteMasterRecord(new MasterLogRecord(999)); + logManager.rewriteMasterRecord(new MasterLogRecord(-1)); + + Iterator iter = logManager.iterator(); + assertEquals(new MasterLogRecord(-1), iter.next()); + for (int i = 1; i < 1000; ++i) { + assertEquals(new MasterLogRecord(i), iter.next()); + } + assertFalse(iter.hasNext()); + } + + @Test + public void testPartialFlush() { + for (int i = 0; i < (DiskSpaceManager.PAGE_SIZE / 9) * 7; ++i) { + logManager.appendToLog(new MasterLogRecord(i)); + } + Page p = bufferManager.fetchPage(new DummyLockContext((LockContext) null), 4L, true); + p.unpin(); + p.flush(); + long prevIO = bufferManager.getNumIOs(); + logManager.flushToLSN(20001); + long postIO = bufferManager.getNumIOs(); + assertEquals(3, postIO - prevIO); + + prevIO = bufferManager.getNumIOs(); + logManager.flushToLSN(50001); + postIO = bufferManager.getNumIOs(); + assertEquals(2, postIO - prevIO); + + prevIO = bufferManager.getNumIOs(); + logManager.flushToLSN(50055); + postIO = bufferManager.getNumIOs(); + assertEquals(0, postIO - prevIO); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/recovery/TestLogRecord.java b/src/test/java/edu/berkeley/cs186/database/recovery/TestLogRecord.java new file mode 100644 index 0000000..10d43e8 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/recovery/TestLogRecord.java @@ -0,0 +1,148 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.categories.SystemTests; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.memory.BufferManager; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +@Category(SystemTests.class) +public class TestLogRecord { + private void checkSerialize(LogRecord record) { + assertEquals(record, LogRecord.fromBytes(ByteBuffer.wrap(record.toBytes())).orElse(null)); + } + + @Test + public void testMasterSerialize() { + checkSerialize(new MasterLogRecord(-98765L)); + } + + @Test + public void testAbortTransactionSerialize() { + checkSerialize(new AbortTransactionLogRecord(-98765L, -43210L)); + } + + @Test + public void testCommitTransactionSerialize() { + checkSerialize(new CommitTransactionLogRecord(-98765L, -43210L)); + } + + @Test + public void testEndTransactionSerialize() { + checkSerialize(new EndTransactionLogRecord(-98765L, -43210L)); + } + + @Test + public void testAllocPageSerialize() { + checkSerialize(new AllocPageLogRecord(-98765L, -43210L, -77654L)); + } + + @Test + public void testFreePageSerialize() { + checkSerialize(new FreePageLogRecord(-98765L, -43210L, -77654L)); + } + + @Test + public void testAllocPartSerialize() { + checkSerialize(new AllocPartLogRecord(-98765L, -43210, -77654L)); + } + + @Test + public void testFreePartSerialize() { + checkSerialize(new FreePartLogRecord(-98765L, -43210, -77654L)); + } + + @Test + public void testUndoAllocPageSerialize() { + checkSerialize(new UndoAllocPageLogRecord(-98765L, -43210L, -77654L, -91235L)); + } + + @Test + public void testUndoFreePageSerialize() { + checkSerialize(new UndoFreePageLogRecord(-98765L, -43210L, -77654L, -91235L)); + } + + @Test + public void testUndoAllocPartSerialize() { + checkSerialize(new UndoAllocPartLogRecord(-98765L, -43210, -77654L, -91235L)); + } + + @Test + public void testUndoFreePartSerialize() { + checkSerialize(new UndoFreePartLogRecord(-98765L, -43210, -77654L, -91235L)); + } + + @Test + public void testUpdatePageSerialize() { + checkSerialize(new UpdatePageLogRecord(-98765L, -43210L, -12345L, (short) 1234, "asdfg".getBytes(), + "zxcvb".getBytes())); + checkSerialize(new UpdatePageLogRecord(-98765L, -43210L, -12345L, (short) 1234, null, + "zxcvb".getBytes())); + checkSerialize(new UpdatePageLogRecord(-98765L, -43210L, -12345L, (short) 1234, "asdfg".getBytes(), + null)); + } + + @Test + public void testUndoUpdatePageSerialize() { + byte[] pageString = new String(new char[BufferManager.EFFECTIVE_PAGE_SIZE]).replace('\0', + 'a').getBytes(); + checkSerialize(new UndoUpdatePageLogRecord(-98765L, -43210L, -12345L, -57812L, (short) 0, + "zxcvb".getBytes())); + checkSerialize(new UndoUpdatePageLogRecord(-98765L, -43210L, -12345L, -57812L, (short) 1234, + "zxcvb".getBytes())); + checkSerialize(new UndoUpdatePageLogRecord(-98765L, -43210L, -12345L, -57812L, (short) 0, + pageString)); + } + + @Test + public void testBeginCheckpointSerialize() { + checkSerialize(new BeginCheckpointLogRecord(92587213L)); + } + + @Test + public void testEndCheckpointSerialize() { + Map dpt = new HashMap<>(); + Map> xacts = new HashMap<>(); + Map> touchedPages = new HashMap<>(); + + checkSerialize(new EndCheckpointLogRecord(dpt, xacts, touchedPages)); + + for (long i = 0; i < 100; ++i) { + dpt.put(i, i); + } + + checkSerialize(new EndCheckpointLogRecord(dpt, xacts, touchedPages)); + + for (long i = 0; i < 53; ++i) { + xacts.put(i, new Pair<>(Transaction.Status.RUNNING, i)); + xacts.put(i, new Pair<>(Transaction.Status.COMMITTING, i)); + xacts.put(i, new Pair<>(Transaction.Status.ABORTING, i)); + } + + checkSerialize(new EndCheckpointLogRecord(dpt, xacts, touchedPages)); + + for (long i = 43; i < 63; ++i) { + touchedPages.put(i, new ArrayList<>()); + for (long j = 43; j < i; ++j) { + touchedPages.get(i).add(j); + } + } + + checkSerialize(new EndCheckpointLogRecord(dpt, xacts, touchedPages)); + + dpt.clear(); + checkSerialize(new EndCheckpointLogRecord(dpt, xacts, touchedPages)); + + xacts.clear(); + checkSerialize(new EndCheckpointLogRecord(dpt, xacts, touchedPages)); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/recovery/TestRecoveryManager.java b/src/test/java/edu/berkeley/cs186/database/recovery/TestRecoveryManager.java new file mode 100644 index 0000000..d618151 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/recovery/TestRecoveryManager.java @@ -0,0 +1,828 @@ +package edu.berkeley.cs186.database.recovery; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.Transaction; +import edu.berkeley.cs186.database.categories.HW5Tests; +import edu.berkeley.cs186.database.categories.HiddenTests; +import edu.berkeley.cs186.database.categories.PublicTests; +import edu.berkeley.cs186.database.common.Pair; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.DiskSpaceManagerImpl; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.LRUEvictionPolicy; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import java.util.*; +import java.util.function.Consumer; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.*; + +@Category(HW5Tests.class) +public class TestRecoveryManager { + private String testDir; + private RecoveryManager recoveryManager; + private final Queue> redoMethods = new ArrayDeque<>(); + + // 1 second per test + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 1000 * TimeoutScaling.factor))); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setup() throws Exception { + testDir = tempFolder.newFolder("test-dir").getAbsolutePath(); + recoveryManager = loadRecoveryManager(testDir); + DummyTransaction.cleanupTransactions(); + LogRecord.onRedoHandler(t -> {}); + } + + @After + public void cleanup() { + recoveryManager.close(); + } + + @Test + @Category(PublicTests.class) + public void testSimpleCommit() throws Exception { + long pageNum = 10000000002L; + short pageOffset = 20; + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + LogManager logManager = getLogManager(recoveryManager); + Map transactionTable = getTransactionTable(recoveryManager); + + Transaction transaction1 = DummyTransaction.create(1L); + recoveryManager.startTransaction(transaction1); + + recoveryManager.logPageWrite(1L, pageNum, pageOffset, before, after); + long LSN1 = recoveryManager.commit(1L); + + assertEquals(LSN1, transactionTable.get(1L).lastLSN); + assertEquals(Transaction.Status.COMMITTING, transactionTable.get(1L).transaction.getStatus()); + + Transaction transaction2 = DummyTransaction.create(2L); + recoveryManager.startTransaction(transaction2); + + long LSN2 = recoveryManager.logPageWrite(2L, pageNum + 1, pageOffset, before, after); + + assertTrue(logManager.getFlushedLSN() + " is not greater than or equal to " + LSN1, + LSN1 <= logManager.getFlushedLSN()); + assertTrue(logManager.getFlushedLSN() + " is not less than " + LSN2, + LSN2 > logManager.getFlushedLSN()); + } + + @Test + @Category(PublicTests.class) + public void testAbort() throws Exception { + long pageNum = 10000000002L; + short pageOffset = 20; + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + Map transactionTable = getTransactionTable(recoveryManager); + + Transaction transaction1 = DummyTransaction.create(1L); + recoveryManager.startTransaction(transaction1); + + recoveryManager.logPageWrite(1L, pageNum, pageOffset, before, after); + + Transaction transaction2 = DummyTransaction.create(2L); + recoveryManager.startTransaction(transaction2); + + recoveryManager.logPageWrite(2L, pageNum + 1, pageOffset, before, after); + + long LSN = recoveryManager.abort(1L); + + assertEquals(LSN, transactionTable.get(1L).lastLSN); + assertEquals(Transaction.Status.ABORTING, transactionTable.get(1L).transaction.getStatus()); + assertEquals(Transaction.Status.RUNNING, transactionTable.get(2L).transaction.getStatus()); + } + + @Test + @Category(PublicTests.class) + public void testEnd() throws Exception { + long pageNum = 10000000002L; + short pageOffset = 20; + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + LogManager logManager = getLogManager(recoveryManager); + Map dirtyPageTable = getDirtyPageTable(recoveryManager); + Map transactionTable = getTransactionTable(recoveryManager); + + Transaction transaction1 = DummyTransaction.create(1L); + recoveryManager.startTransaction(transaction1); + Transaction transaction2 = DummyTransaction.create(2L); + recoveryManager.startTransaction(transaction2); + + long[] LSNs = new long[] { + recoveryManager.logPageWrite(1L, pageNum, pageOffset, before, after), // 0 + recoveryManager.logPageWrite(2L, pageNum + 1, pageOffset, before, after), // 1 + recoveryManager.logPageWrite(1L, pageNum, pageOffset, after, before), // 2 + recoveryManager.logPageWrite(2L, pageNum + 1, pageOffset, after, before), // 3 + recoveryManager.logAllocPart(2L, 2), // 4 + recoveryManager.logPageWrite(1L, pageNum, pageOffset, before, after), // 5 + recoveryManager.commit(2L), // 6 + recoveryManager.logPageWrite(1L, pageNum, pageOffset, before, after), // 7 + -1L, + -1L, + }; + + assertEquals(LSNs[7], transactionTable.get(1L).lastLSN); + assertEquals(LSNs[6], transactionTable.get(2L).lastLSN); + assertEquals(LogManagerImpl.maxLSN(LogManagerImpl.getLSNPage(LSNs[6])), logManager.getFlushedLSN()); + assertEquals(Transaction.Status.COMMITTING, transactionTable.get(2L).transaction.getStatus()); + + LSNs[8] = recoveryManager.end(2L); + LSNs[9] = recoveryManager.abort(1L); + + assertEquals(LSNs[9], transactionTable.get(1L).lastLSN); + assertEquals(Transaction.Status.ABORTING, transactionTable.get(1L).transaction.getStatus()); + + recoveryManager.end(1L); // 4 CLRs + END + + Iterator iter = logManager.iterator(); + + // CLRs are written correctly + int totalRecords = 0; // 18 - 3 (master + begin/end chkpt) + 10 (LSNs) + 4 (CLRs) + 1 (END) + int abort = 0; // 1 + int commit = 0; // 1 + int end = 0; // 2 + int update = 0; // 4 + 2 + int allocPart = 0; // 1 + int undo = 0; // 4 + while (iter.hasNext()) { + LogRecord record = iter.next(); + totalRecords++; + switch (record.getType()) { + case ABORT_TRANSACTION: + abort++; + break; + case COMMIT_TRANSACTION: + commit++; + break; + case END_TRANSACTION: + end++; + break; + case UPDATE_PAGE: + update++; + break; + case UNDO_UPDATE_PAGE: + undo++; + break; + case ALLOC_PART: + allocPart++; + break; + } + } + assertEquals(18, totalRecords); + assertEquals(1, abort); + assertEquals(1, commit); + assertEquals(2, end); + assertEquals(6, update); + assertEquals(1, allocPart); + assertEquals(4, undo); + + // Dirty page table + assertEquals(LSNs[0], (long) dirtyPageTable.get(pageNum)); + assertEquals(LSNs[1], (long) dirtyPageTable.get(pageNum + 1)); + + // Transaction table + assertTrue(transactionTable.isEmpty()); + + // Flushed log tail correct + assertEquals(LogManagerImpl.maxLSN(LogManagerImpl.getLSNPage(LSNs[6])), logManager.getFlushedLSN()); + + assertEquals(Transaction.Status.COMPLETE, transaction1.getStatus()); + assertEquals(Transaction.Status.COMPLETE, transaction2.getStatus()); + } + + @Test + @Category(PublicTests.class) + public void testSimpleLogPageWrite() throws Exception { + short pageOffset = 20; + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + Map dirtyPageTable = getDirtyPageTable(recoveryManager); + Map transactionTable = getTransactionTable(recoveryManager); + + Transaction transaction1 = DummyTransaction.create(1L); + recoveryManager.startTransaction(transaction1); + + long LSN1 = recoveryManager.logPageWrite(transaction1.getTransNum(), 10000000002L, pageOffset, + before, after); + + // Check DPT, X-Act table updates + assertTrue(transactionTable.containsKey(1L)); + assertEquals(LSN1, transactionTable.get(1L).lastLSN); + assertTrue(transactionTable.get(1L).touchedPages.contains(10000000002L)); + assertTrue(dirtyPageTable.containsKey(10000000002L)); + assertEquals(LSN1, (long) dirtyPageTable.get(10000000002L)); + + Transaction transaction2 = DummyTransaction.create(2L); + recoveryManager.startTransaction(transaction2); + + long LSN2 = recoveryManager.logPageWrite(transaction2.getTransNum(), 10000000003L, pageOffset, + before, after); + + assertTrue(transactionTable.containsKey(2L)); + assertEquals(LSN2, transactionTable.get(2L).lastLSN); + assertTrue(transactionTable.get(2L).touchedPages.contains(10000000003L)); + assertEquals(LSN2, (long) dirtyPageTable.get(10000000003L)); + + long LSN3 = recoveryManager.logPageWrite(transaction1.getTransNum(), 10000000002L, pageOffset, + before, after); + assertEquals(LSN3, transactionTable.get(1L).lastLSN); + assertEquals(LSN1, (long) dirtyPageTable.get(10000000002L)); + } + + @Test + @Category(PublicTests.class) + public void testTwoPartLogPageWrite() throws Exception { + long pageNum = 10000000002L; + byte[] before = new byte[BufferManager.EFFECTIVE_PAGE_SIZE]; + byte[] after = new byte[BufferManager.EFFECTIVE_PAGE_SIZE]; + for (int i = 0; i < BufferManager.EFFECTIVE_PAGE_SIZE; ++i) { + after[i] = (byte) (i % 256); + } + + LogManager logManager = getLogManager(recoveryManager); + Map transactionTable = getTransactionTable(recoveryManager); + + Transaction transaction1 = DummyTransaction.create(1L); + recoveryManager.startTransaction(transaction1); + + long secondLSN = recoveryManager.logPageWrite(transaction1.getTransNum(), pageNum, (short) 0, + before, after); + long firstLSN = secondLSN - 10000L; // previous log page, both at start of a log page + + LogRecord firstLogRecord = logManager.fetchLogRecord(firstLSN); + LogRecord secondLogRecord = logManager.fetchLogRecord(secondLSN); + + assertTrue(firstLogRecord instanceof UpdatePageLogRecord); + assertTrue(secondLogRecord instanceof UpdatePageLogRecord); + assertArrayEquals(before, ((UpdatePageLogRecord) firstLogRecord).before); + assertArrayEquals(new byte[0], ((UpdatePageLogRecord) secondLogRecord).before); + assertArrayEquals(new byte[0], ((UpdatePageLogRecord) firstLogRecord).after); + assertArrayEquals(after, ((UpdatePageLogRecord) secondLogRecord).after); + + assertTrue(transactionTable.containsKey(1L)); + assertTrue(transactionTable.get(1L).touchedPages.contains(pageNum)); + } + + @Test + @Category(PublicTests.class) + public void testSimpleSavepoint() throws Exception { + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + LogManager logManager = getLogManager(recoveryManager); + + Transaction transaction1 = DummyTransaction.create(1L); + recoveryManager.startTransaction(transaction1); + recoveryManager.savepoint(1L, "savepoint 1"); + long LSN = recoveryManager.logPageWrite(1L, 10000000001L, (short) 0, before, after); + + recoveryManager.rollbackToSavepoint(1L, "savepoint 1"); + + Iterator iter = logManager.scanFrom(LSN); + iter.next(); // page write record + + LogRecord clr = iter.next(); + assertEquals(LogType.UNDO_UPDATE_PAGE, clr.getType()); + + assertEquals((short) 0, ((UndoUpdatePageLogRecord) clr).offset); + assertArrayEquals(before, ((UndoUpdatePageLogRecord) clr).after); + assertEquals(Optional.of(1L), clr.getTransNum()); + assertEquals(Optional.of(10000000001L), clr.getPageNum()); + assertTrue(clr.getUndoNextLSN().orElseThrow(NoSuchElementException::new) < LSN); + } + + @Test + @Category(PublicTests.class) + public void testSimpleCheckpoint() throws Exception { + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + LogManager logManager = getLogManager(recoveryManager); + + Transaction transaction1 = DummyTransaction.create(1L); + recoveryManager.startTransaction(transaction1); + + long LSN1 = recoveryManager.logPageWrite(1L, 10000000001L, (short) 0, before, after); + + recoveryManager.checkpoint(); + + recoveryManager.logPageWrite(1L, 10000000001L, (short) 0, after, before); + recoveryManager.logPageWrite(1L, 10000000001L, (short) 0, before, after); + + Iterator iter = logManager.scanFrom(LSN1); + iter.next(); // page write (LSN 1) + + LogRecord beginCheckpoint = iter.next(); + LogRecord endCheckpoint = iter.next(); + assertEquals(LogType.BEGIN_CHECKPOINT, beginCheckpoint.getType()); + assertEquals(LogType.END_CHECKPOINT, endCheckpoint.getType()); + + Map> txnTable = endCheckpoint.getTransactionTable(); + Map dpt = endCheckpoint.getDirtyPageTable(); + assertEquals(LSN1, (long) dpt.get(10000000001L)); + assertEquals(new Pair<>(Transaction.Status.RUNNING, LSN1), txnTable.get(1L)); + } + + @Test + @Category(PublicTests.class) + public void testRestartAnalysis() throws Exception { + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + LogManager logManager = getLogManager(recoveryManager); + + DummyTransaction transaction1 = DummyTransaction.create(1L); + DummyTransaction transaction2 = DummyTransaction.create(2L); + DummyTransaction transaction3 = DummyTransaction.create(3L); + + List LSNs = new ArrayList<>(); + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000001L, 0L, (short) 0, before, + after))); // 0 + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000002L, LSNs.get(0), (short) 0, + before, after))); // 1 + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(3L, 10000000003L, 0L, (short) 0, + before, after))); // 2 + LSNs.add(logManager.appendToLog(new CommitTransactionLogRecord(1L, LSNs.get(1)))); // 3 + LSNs.add(logManager.appendToLog(new EndTransactionLogRecord(1L, LSNs.get(3)))); // 4 + LSNs.add(logManager.appendToLog(new FreePageLogRecord(2L, 10000000001L, 0L))); // 5 + LSNs.add(logManager.appendToLog(new AbortTransactionLogRecord(2L, LSNs.get(5)))); // 6 + LSNs.add(logManager.appendToLog(new BeginCheckpointLogRecord(9876543210L))); // 7 + + // flush everything - recovery tests should always start + // with a clean load from disk, and here we want everything sent to disk first. + // Note: this does not call RecoveryManager#close - it only closes the + // buffer manager and disk space manager. + shutdownRecoveryManager(recoveryManager); + + // load from disk again + recoveryManager = loadRecoveryManager(testDir); + + // new recovery manager - tables/log manager/other state loaded with old manager are different + // with the new recovery manager + logManager = getLogManager(recoveryManager); + Map dirtyPageTable = getDirtyPageTable(recoveryManager); + Map transactionTable = getTransactionTable(recoveryManager); + List lockRequests = getLockRequests(recoveryManager); + + runAnalysis(recoveryManager); + + // Xact table + assertFalse(transactionTable.containsKey(1L)); + assertTrue(transactionTable.containsKey(2L)); + assertEquals((long) LSNs.get(6), transactionTable.get(2L).lastLSN); + assertEquals(new HashSet<>(Collections.singletonList(10000000001L)), + transactionTable.get(2L).touchedPages); + assertTrue(transactionTable.containsKey(3L)); + assertTrue(transactionTable.get(3L).lastLSN > LSNs.get(7)); + assertEquals(new HashSet<>(Collections.singletonList(10000000003L)), + transactionTable.get(3L).touchedPages); + + // DPT + assertFalse(dirtyPageTable.containsKey(10000000001L)); + assertTrue(dirtyPageTable.containsKey(10000000002L)); + assertEquals((long) LSNs.get(1), (long) dirtyPageTable.get(10000000002L)); + assertTrue(dirtyPageTable.containsKey(10000000003L)); + assertEquals((long) LSNs.get(2), (long) dirtyPageTable.get(10000000003L)); + + // status/cleanup + assertEquals(Transaction.Status.COMPLETE, transaction1.getStatus()); + assertTrue(transaction1.cleanedUp); + assertEquals(Transaction.Status.RECOVERY_ABORTING, transaction2.getStatus()); + assertFalse(transaction2.cleanedUp); + assertEquals(Transaction.Status.RECOVERY_ABORTING, transaction3.getStatus()); + assertFalse(transaction2.cleanedUp); + + // lock requests made + assertEquals(Arrays.asList( + "request 1 X(database/1/10000000001)", + "request 1 X(database/1/10000000002)", + "request 3 X(database/1/10000000003)", + "request 2 X(database/1/10000000001)" + ), lockRequests); + + // transaction counter - from begin checkpoint + assertEquals(9876543210L, getTransactionCounter(recoveryManager)); + + // FlushedLSN + assertEquals(LogManagerImpl.maxLSN(LogManagerImpl.getLSNPage(LSNs.get(7))), + logManager.getFlushedLSN()); + } + + @Test + @Category(PublicTests.class) + public void testRestartRedo() throws Exception { + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + LogManager logManager = getLogManager(recoveryManager); + DiskSpaceManager dsm = getDiskSpaceManager(recoveryManager); + BufferManager bm = getBufferManager(recoveryManager); + + DummyTransaction transaction1 = DummyTransaction.create(1L); + + List LSNs = new ArrayList<>(); + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000001L, 0L, (short) 0, before, + after))); // 0 + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000002L, LSNs.get(0), (short) 1, + before, after))); // 1 + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000002L, LSNs.get(1), (short) 1, + after, before))); // 2 + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000003L, LSNs.get(2), (short) 2, + before, after))); // 3 + LSNs.add(logManager.appendToLog(new AllocPartLogRecord(1L, 10, LSNs.get(3)))); // 4 + LSNs.add(logManager.appendToLog(new CommitTransactionLogRecord(1L, LSNs.get(4)))); // 5 + LSNs.add(logManager.appendToLog(new EndTransactionLogRecord(1L, LSNs.get(5)))); // 6 + + // actually do the first and second write (and get it flushed to disk) + logManager.fetchLogRecord(LSNs.get(0)).redo(dsm, bm); + logManager.fetchLogRecord(LSNs.get(1)).redo(dsm, bm); + + // flush everything - recovery tests should always start + // with a clean load from disk, and here we want everything sent to disk first. + // Note: this does not call RecoveryManager#close - it only closes the + // buffer manager and disk space manager. + shutdownRecoveryManager(recoveryManager); + + // load from disk again + recoveryManager = loadRecoveryManager(testDir); + + // set up dirty page table - xact table is empty (transaction ended) + Map dirtyPageTable = getDirtyPageTable(recoveryManager); + dirtyPageTable.put(10000000002L, LSNs.get(2)); + dirtyPageTable.put(10000000003L, LSNs.get(3)); + + // set up checks for redo - these get called in sequence with each LogRecord#redo call + setupRedoChecks(Arrays.asList( + (LogRecord record) -> assertEquals((long) LSNs.get(2), (long) record.LSN), + (LogRecord record) -> assertEquals((long) LSNs.get(3), (long) record.LSN), + (LogRecord record) -> assertEquals((long) LSNs.get(4), (long) record.LSN) + )); + + runRedo(recoveryManager); + + finishRedoChecks(); + } + + @Test + @Category(PublicTests.class) + public void testRestartUndo() throws Exception { + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + LogManager logManager = getLogManager(recoveryManager); + DiskSpaceManager dsm = getDiskSpaceManager(recoveryManager); + BufferManager bm = getBufferManager(recoveryManager); + + DummyTransaction transaction1 = DummyTransaction.create(1L); + + List LSNs = new ArrayList<>(); + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000001L, 0L, (short) 0, before, + after))); // 0 + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000002L, LSNs.get(0), (short) 1, + before, after))); // 1 + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000003L, LSNs.get(1), (short) 2, + before, after))); // 2 + LSNs.add(logManager.appendToLog(new UpdatePageLogRecord(1L, 10000000004L, LSNs.get(2), (short) 3, + before, after))); // 3 + LSNs.add(logManager.appendToLog(new AbortTransactionLogRecord(1L, LSNs.get(3)))); // 4 + + // actually do the writes + for (int i = 0; i < 4; ++i) { + logManager.fetchLogRecord(LSNs.get(i)).redo(dsm, bm); + } + + // flush everything - recovery tests should always start + // with a clean load from disk, and here we want everything sent to disk first. + // Note: this does not call RecoveryManager#close - it only closes the + // buffer manager and disk space manager. + shutdownRecoveryManager(recoveryManager); + + // load from disk again + recoveryManager = loadRecoveryManager(testDir); + + // set up xact table - leaving DPT empty + Map transactionTable = getTransactionTable(recoveryManager); + TransactionTableEntry entry1 = new TransactionTableEntry(transaction1); + entry1.lastLSN = LSNs.get(4); + entry1.touchedPages = new HashSet<>(Arrays.asList(10000000001L, 10000000002L, 10000000003L, + 10000000004L)); + entry1.transaction.setStatus(Transaction.Status.RECOVERY_ABORTING); + transactionTable.put(1L, entry1); + + // set up checks for undo - these get called in sequence with each LogRecord#redo call + // (which should be called on CLRs) + setupRedoChecks(Arrays.asList( + (LogRecord record) -> { + assertEquals(LogType.UNDO_UPDATE_PAGE, record.getType()); + assertNotNull("log record not appended to log yet", record.LSN); + assertEquals((long) record.LSN, transactionTable.get(1L).lastLSN); + assertEquals(Optional.of(10000000004L), record.getPageNum()); + }, + (LogRecord record) -> { + assertEquals(LogType.UNDO_UPDATE_PAGE, record.getType()); + assertNotNull("log record not appended to log yet", record.LSN); + assertEquals((long) record.LSN, transactionTable.get(1L).lastLSN); + assertEquals(Optional.of(10000000003L), record.getPageNum()); + }, + (LogRecord record) -> { + assertEquals(LogType.UNDO_UPDATE_PAGE, record.getType()); + assertNotNull("log record not appended to log yet", record.LSN); + assertEquals((long) record.LSN, transactionTable.get(1L).lastLSN); + assertEquals(Optional.of(10000000002L), record.getPageNum()); + }, + (LogRecord record) -> { + assertEquals(LogType.UNDO_UPDATE_PAGE, record.getType()); + assertNotNull("log record not appended to log yet", record.LSN); + assertEquals((long) record.LSN, transactionTable.get(1L).lastLSN); + assertEquals(Optional.of(10000000001L), record.getPageNum()); + } + )); + + runUndo(recoveryManager); + + finishRedoChecks(); + + assertEquals(Transaction.Status.COMPLETE, transaction1.getStatus()); + } + + @Test + @Category(PublicTests.class) + public void testSimpleRestart() throws Exception { + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + Transaction transaction1 = DummyTransaction.create(1L); + + recoveryManager.startTransaction(transaction1); + long LSN = recoveryManager.logPageWrite(1L, 10000000001L, (short) 0, before, after); + + // flush everything - recovery tests should always start + // with a clean load from disk, and here we want everything sent to disk first. + // Note: this does not call RecoveryManager#close - it only closes the + // buffer manager and disk space manager. + shutdownRecoveryManager(recoveryManager); + + // load from disk again + recoveryManager = loadRecoveryManager(testDir); + + LogManager logManager = getLogManager(recoveryManager); + Map dirtyPageTable = getDirtyPageTable(recoveryManager); + Map transactionTable = getTransactionTable(recoveryManager); + + setupRedoChecks(Collections.singletonList( + (LogRecord record) -> { + assertEquals(LSN, record.getLSN()); + assertEquals(LogType.UPDATE_PAGE, record.getType()); + } + )); + + Runnable func = recoveryManager.restart(); // analysis + redo + + finishRedoChecks(); + + assertTrue(transactionTable.containsKey(transaction1.getTransNum())); + TransactionTableEntry entry = transactionTable.get(transaction1.getTransNum()); + assertEquals(Transaction.Status.RECOVERY_ABORTING, entry.transaction.getStatus()); + assertEquals(new HashSet<>(Collections.singletonList(10000000001L)), entry.touchedPages); + assertEquals(LSN, (long) dirtyPageTable.get(10000000001L)); + + func.run(); // undo + + Iterator iter = logManager.scanFrom(LSN); + + assertEquals(LogType.UPDATE_PAGE, iter.next().getType()); + assertEquals(LogType.ABORT_TRANSACTION, iter.next().getType()); + assertEquals(LogType.UNDO_UPDATE_PAGE, iter.next().getType()); + assertEquals(LogType.END_TRANSACTION, iter.next().getType()); + assertEquals(LogType.BEGIN_CHECKPOINT, iter.next().getType()); + assertEquals(LogType.END_CHECKPOINT, iter.next().getType()); + assertFalse(iter.hasNext()); + } + + @Test + @Category(PublicTests.class) + public void testRestart() throws Exception { + byte[] before = new byte[] { (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00 }; + byte[] after = new byte[] { (byte) 0xBA, (byte) 0xAD, (byte) 0xF0, (byte) 0x0D }; + + Transaction transaction1 = DummyTransaction.create(1L); + recoveryManager.startTransaction(transaction1); + Transaction transaction2 = DummyTransaction.create(2L); + recoveryManager.startTransaction(transaction2); + Transaction transaction3 = DummyTransaction.create(3L); + recoveryManager.startTransaction(transaction3); + + long[] LSNs = new long[] { + recoveryManager.logPageWrite(1L, 10000000001L, (short) 0, before, after), // 0 + recoveryManager.logPageWrite(2L, 10000000003L, (short) 0, before, after), // 1 + recoveryManager.commit(1L), // 2 + recoveryManager.logPageWrite(3L, 10000000004L, (short) 0, before, after), // 3 + recoveryManager.logPageWrite(2L, 10000000001L, (short) 0, after, before), // 4 + recoveryManager.end(1L), // 5 + recoveryManager.logPageWrite(3L, 10000000002L, (short) 0, before, after), // 6 + recoveryManager.abort(2), // 7 + }; + + // flush everything - recovery tests should always start + // with a clean load from disk, and here we want everything sent to disk first. + // Note: this does not call RecoveryManager#close - it only closes the + // buffer manager and disk space manager. + shutdownRecoveryManager(recoveryManager); + + // load from disk again + recoveryManager = loadRecoveryManager(testDir); + + LogManager logManager = getLogManager(recoveryManager); + + recoveryManager.restart().run(); // run everything in restart recovery + + Iterator iter = logManager.iterator(); + assertEquals(LogType.MASTER, iter.next().getType()); + assertEquals(LogType.BEGIN_CHECKPOINT, iter.next().getType()); + assertEquals(LogType.END_CHECKPOINT, iter.next().getType()); + assertEquals(LogType.UPDATE_PAGE, iter.next().getType()); + assertEquals(LogType.UPDATE_PAGE, iter.next().getType()); + assertEquals(LogType.COMMIT_TRANSACTION, iter.next().getType()); + assertEquals(LogType.UPDATE_PAGE, iter.next().getType()); + assertEquals(LogType.UPDATE_PAGE, iter.next().getType()); + assertEquals(LogType.END_TRANSACTION, iter.next().getType()); + assertEquals(LogType.UPDATE_PAGE, iter.next().getType()); + assertEquals(LogType.ABORT_TRANSACTION, iter.next().getType()); + + LogRecord record = iter.next(); + assertEquals(LogType.ABORT_TRANSACTION, record.getType()); + long LSN8 = record.LSN; + + record = iter.next(); + assertEquals(LogType.UNDO_UPDATE_PAGE, record.getType()); + assertEquals(LSN8, (long) record.getPrevLSN().orElseThrow(NoSuchElementException::new)); + assertEquals(LSNs[3], (long) record.getUndoNextLSN().orElseThrow(NoSuchElementException::new)); + long LSN9 = record.LSN; + + record = iter.next(); + assertEquals(LogType.UNDO_UPDATE_PAGE, record.getType()); + assertEquals(LSNs[7], (long) record.getPrevLSN().orElseThrow(NoSuchElementException::new)); + assertEquals(LSNs[1], (long) record.getUndoNextLSN().orElseThrow(NoSuchElementException::new)); + long LSN10 = record.LSN; + + record = iter.next(); + assertEquals(LogType.UNDO_UPDATE_PAGE, record.getType()); + assertEquals(LSN9, (long) record.getPrevLSN().orElseThrow(NoSuchElementException::new)); + assertEquals(0L, (long) record.getUndoNextLSN().orElseThrow(NoSuchElementException::new)); + assertEquals(LogType.END_TRANSACTION, iter.next().getType()); + + record = iter.next(); + assertEquals(LogType.UNDO_UPDATE_PAGE, record.getType()); + assertEquals(LSN10, (long) record.getPrevLSN().orElseThrow(NoSuchElementException::new)); + assertEquals(0L, (long) record.getUndoNextLSN().orElseThrow(NoSuchElementException::new)); + + assertEquals(LogType.END_TRANSACTION, iter.next().getType()); + assertEquals(LogType.BEGIN_CHECKPOINT, iter.next().getType()); + assertEquals(LogType.END_CHECKPOINT, iter.next().getType()); + assertFalse(iter.hasNext()); + } + + /************************************************************************* + * Helpers - these are similar to the ones available in TestARIESStudent * + *************************************************************************/ + + /** + * Helper to set up checks for redo. The first call to LogRecord.redo will + * call the first method in METHODS, the second call to the second method in METHODS, + * and so on. Call this method before the redo pass, and call finishRedoChecks + * after the redo pass. + */ + private void setupRedoChecks(Collection> methods) { + for (final Consumer method : methods) { + redoMethods.add(record -> { + method.accept(record); + LogRecord.onRedoHandler(redoMethods.poll()); + }); + } + redoMethods.add(record -> fail("LogRecord#redo() called too many times")); + LogRecord.onRedoHandler(redoMethods.poll()); + } + + /** + * Helper to finish checks for redo. Call this after the redo pass (or undo pass)- + * if not enough redo calls were performed, an error is thrown. + * + * If setupRedoChecks is used for the redo pass, and this method is not called before + * the undo pass, and the undo pass calls undo at least once, an error may be incorrectly thrown. + */ + private void finishRedoChecks() { + assertTrue("LogRecord#redo() not called enough times", redoMethods.isEmpty()); + LogRecord.onRedoHandler(record -> {}); + } + + /** + * Loads the recovery manager from disk. + * @param dir testDir + * @return recovery manager, loaded from disk + */ + protected RecoveryManager loadRecoveryManager(String dir) throws Exception { + RecoveryManager recoveryManager = new ARIESRecoveryManagerNoLocking( + new DummyLockContext(new Pair<>("database", 0L)), + DummyTransaction::create + ); + DiskSpaceManager diskSpaceManager = new DiskSpaceManagerImpl(dir, recoveryManager); + BufferManager bufferManager = new BufferManagerImpl(diskSpaceManager, recoveryManager, 32, + new LRUEvictionPolicy()); + boolean isLoaded = true; + try { + diskSpaceManager.allocPart(0); + diskSpaceManager.allocPart(1); + for (int i = 0; i < 10; ++i) { + diskSpaceManager.allocPage(DiskSpaceManager.getVirtualPageNum(1, i)); + } + isLoaded = false; + } catch (IllegalStateException e) { + // already loaded + } + recoveryManager.setManagers(diskSpaceManager, bufferManager); + if (!isLoaded) { + recoveryManager.initialize(); + } + return recoveryManager; + } + + /** + * Flushes everything to disk, but does not call RecoveryManager#shutdown. Similar + * to pulling the plug on the database at a time when no changes are in memory. You + * can simulate a shutdown where certain changes _are_ in memory, by simply never + * applying them (i.e. write a log record, but do not make the changes on the + * buffer manager/disk space manager). + */ + protected void shutdownRecoveryManager(RecoveryManager recoveryManager) throws Exception { + ARIESRecoveryManager arm = (ARIESRecoveryManager) recoveryManager; + arm.logManager.close(); + arm.bufferManager.evictAll(); + arm.bufferManager.close(); + arm.diskSpaceManager.close(); + DummyTransaction.cleanupTransactions(); + } + + protected BufferManager getBufferManager(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).bufferManager; + } + + protected DiskSpaceManager getDiskSpaceManager(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).diskSpaceManager; + } + + protected LogManager getLogManager(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).logManager; + } + + protected List getLockRequests(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).lockRequests; + } + + protected long getTransactionCounter(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManagerNoLocking) recoveryManager).transactionCounter; + } + + protected Map getDirtyPageTable(RecoveryManager recoveryManager) throws Exception { + return ((ARIESRecoveryManager) recoveryManager).dirtyPageTable; + } + + protected Map getTransactionTable(RecoveryManager recoveryManager) + throws Exception { + return ((ARIESRecoveryManager) recoveryManager).transactionTable; + } + + protected void runAnalysis(RecoveryManager recoveryManager) throws Exception { + ((ARIESRecoveryManager) recoveryManager).restartAnalysis(); + } + + protected void runRedo(RecoveryManager recoveryManager) throws Exception { + ((ARIESRecoveryManager) recoveryManager).restartRedo(); + } + + protected void runUndo(RecoveryManager recoveryManager) throws Exception { + ((ARIESRecoveryManager) recoveryManager).restartUndo(); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/table/MemoryHeapFile.java b/src/test/java/edu/berkeley/cs186/database/table/MemoryHeapFile.java new file mode 100644 index 0000000..78519dd --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/table/MemoryHeapFile.java @@ -0,0 +1,123 @@ +package edu.berkeley.cs186.database.table; + +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.common.iterator.IndexBacktrackingIterator; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.MemoryDiskSpaceManager; +import edu.berkeley.cs186.database.io.PageException; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.ClockEvictionPolicy; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.recovery.DummyRecoveryManager; + +import java.util.*; + +/** + * Heap file implementation that is entirely in memory. Not thread safe. + */ +public class MemoryHeapFile implements HeapFile, AutoCloseable { + private List pageNums = new ArrayList<>(); + private Map pages = new HashMap<>(); + private Map freeSpace = new HashMap<>(); + private short emptyPageMetadataSize = 0; + private BufferManager bufferManager; + private int numDataPages = 0; + + public MemoryHeapFile() { + DiskSpaceManager diskSpaceManager = new MemoryDiskSpaceManager(); + diskSpaceManager.allocPart(0); + this.bufferManager = new BufferManagerImpl(diskSpaceManager, new DummyRecoveryManager(), 1024, + new ClockEvictionPolicy()); + } + + @Override + public void close() { + this.bufferManager.close(); + } + + @Override + public short getEffectivePageSize() { + return BufferManager.EFFECTIVE_PAGE_SIZE; + } + + @Override + public void setEmptyPageMetadataSize(short emptyPageMetadataSize) { + this.emptyPageMetadataSize = emptyPageMetadataSize; + } + + @Override + public Page getPage(long pageNum) { + if (!pages.containsKey(pageNum) || pages.get(pageNum) == null) { + throw new PageException("page not found"); + } + return bufferManager.fetchPage(new DummyLockContext(), pageNum, false); + } + + @Override + public Page getPageWithSpace(short requiredSpace) { + for (Map.Entry entry : freeSpace.entrySet()) { + if (entry.getValue() >= requiredSpace) { + freeSpace.put(entry.getKey(), (short) (entry.getValue() - requiredSpace)); + return bufferManager.fetchPage(new DummyLockContext(), entry.getKey(), false); + } + } + Page page = bufferManager.fetchNewPage(new DummyLockContext(), 0, false); + pageNums.add(page.getPageNum()); + pages.put(page.getPageNum(), page); + freeSpace.put(page.getPageNum(), + (short) (getEffectivePageSize() - emptyPageMetadataSize - requiredSpace)); + ++numDataPages; + return page; + } + + @Override + public void updateFreeSpace(Page page, short newFreeSpace) { + if (newFreeSpace == getEffectivePageSize() - emptyPageMetadataSize) { + pages.put(page.getPageNum(), null); + --numDataPages; + } + freeSpace.put(page.getPageNum(), newFreeSpace); + } + + @Override + public BacktrackingIterator iterator() { + return new PageIterator(); + } + + @Override + public int getNumDataPages() { + return numDataPages; + } + + @Override + public int getPartNum() { + return 0; + } + + private class PageIterator extends IndexBacktrackingIterator { + PageIterator() { + super(pageNums.size()); + } + + @Override + protected int getNextNonempty(int currentIndex) { + ++currentIndex; + while (currentIndex < pageNums.size()) { + Page page = getValue(currentIndex); + if (page != null) { + page.unpin(); + break; + } + ++currentIndex; + } + return currentIndex; + } + + @Override + protected Page getValue(int index) { + return bufferManager.fetchPage(new DummyLockContext(), pageNums.get(index), false); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/table/TestPageDirectory.java b/src/test/java/edu/berkeley/cs186/database/table/TestPageDirectory.java new file mode 100644 index 0000000..63eb4ba --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/table/TestPageDirectory.java @@ -0,0 +1,250 @@ +package edu.berkeley.cs186.database.table; + +import edu.berkeley.cs186.database.categories.HW99Tests; +import edu.berkeley.cs186.database.categories.SystemTests; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.io.DiskSpaceManager; +import edu.berkeley.cs186.database.io.MemoryDiskSpaceManager; +import edu.berkeley.cs186.database.io.PageException; +import edu.berkeley.cs186.database.memory.BufferManager; +import edu.berkeley.cs186.database.memory.BufferManagerImpl; +import edu.berkeley.cs186.database.memory.ClockEvictionPolicy; +import edu.berkeley.cs186.database.memory.Page; +import edu.berkeley.cs186.database.recovery.DummyRecoveryManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.*; + +import static org.junit.Assert.*; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestPageDirectory { + private BufferManager bufferManager; + private PageDirectory pageDirectory; + + @Before + public void setup() { + DiskSpaceManager diskSpaceManager = new MemoryDiskSpaceManager(); + diskSpaceManager.allocPart(0); + this.bufferManager = new BufferManagerImpl(diskSpaceManager, new DummyRecoveryManager(), 1024, + new ClockEvictionPolicy()); + pageDirectory = null; + } + + @After + public void cleanup() { + bufferManager.close(); + } + + private void createPageDirectory(long pageNum, short metadataSize) { + pageDirectory = new PageDirectory(bufferManager, 0, pageNum, metadataSize, new DummyLockContext()); + } + + private void createPageDirectory(short metadataSize) { + Page page = bufferManager.fetchNewPage(new DummyLockContext(), 0, false); + try { + createPageDirectory(page.getPageNum(), metadataSize); + } finally { + page.unpin(); + } + } + + @Test + public void testGetPageWithSpace() { + createPageDirectory((short) 10); + + short pageSize = (short) (pageDirectory.getEffectivePageSize() - 10); + Page p1 = pageDirectory.getPageWithSpace(pageSize); + Page p2 = pageDirectory.getPageWithSpace((short) 1); + Page p3 = pageDirectory.getPageWithSpace((short) 60); + Page p4 = pageDirectory.getPageWithSpace((short) (pageSize - 60)); + Page p5 = pageDirectory.getPageWithSpace((short) 120); + + p1.unpin(); p2.unpin(); p3.unpin(); p4.unpin(); p5.unpin(); + + assertNotEquals(p1, p2); + assertEquals(p2, p3); + assertEquals(p3, p5); + assertNotEquals(p4, p5); + } + + @Test + public void testGetPageWithSpaceInvalid() { + createPageDirectory((short) 1000); + + short pageSize = (short) (pageDirectory.getEffectivePageSize() - 1000); + + try { + pageDirectory.getPageWithSpace((short) (pageSize + 1)); + fail(); + } catch (IllegalArgumentException e) { /* do nothing */ } + + try { + pageDirectory.getPageWithSpace((short) 0); + fail(); + } catch (IllegalArgumentException e) { /* do nothing */ } + + try { + pageDirectory.getPageWithSpace((short) - 1); + fail(); + } catch (IllegalArgumentException e) { /* do nothing */ } + } + + @Test + public void testGetPage() { + createPageDirectory((short) 10); + + short pageSize = (short) (pageDirectory.getEffectivePageSize() - 10); + Page p1 = pageDirectory.getPageWithSpace(pageSize); + Page p2 = pageDirectory.getPageWithSpace((short) 1); + Page p3 = pageDirectory.getPageWithSpace((short) 60); + Page p4 = pageDirectory.getPageWithSpace((short) (pageSize - 60)); + Page p5 = pageDirectory.getPageWithSpace((short) 120); + + p1.unpin(); p2.unpin(); p3.unpin(); p4.unpin(); p5.unpin(); + + Page pp1 = pageDirectory.getPage(p1.getPageNum()); + Page pp2 = pageDirectory.getPage(p2.getPageNum()); + Page pp3 = pageDirectory.getPage(p3.getPageNum()); + Page pp4 = pageDirectory.getPage(p4.getPageNum()); + Page pp5 = pageDirectory.getPage(p5.getPageNum()); + + pp1.unpin(); pp2.unpin(); pp3.unpin(); pp4.unpin(); pp5.unpin(); + + assertEquals(p1, pp1); + assertEquals(p2, pp2); + assertEquals(p3, pp3); + assertEquals(p4, pp4); + assertEquals(p5, pp5); + } + + @Test + public void testGetPageInvalid() { + createPageDirectory((short) 10); + Page p = pageDirectory.getPageWithSpace((short) 1); + p.unpin(); + + createPageDirectory((short) 10); + try { + pageDirectory.getPage(p.getPageNum()); + fail(); + } catch (PageException e) { /* do nothing */ } + + try { + pageDirectory.getPage(DiskSpaceManager.INVALID_PAGE_NUM); + fail(); + } catch (PageException e) { /* do nothing */ } + } + + @Test + public void testUpdateFreeSpace() { + createPageDirectory((short) 10); + + short pageSize = (short) (pageDirectory.getEffectivePageSize() - 10); + Page p1 = pageDirectory.getPageWithSpace(pageSize); + p1.unpin(); + + pageDirectory.updateFreeSpace(p1, (short) 10); + + Page p2 = pageDirectory.getPageWithSpace((short) 10); + p2.unpin(); + + assertEquals(p1, p2); + } + + @Test + public void testUpdateFreeSpaceInvalid1() { + createPageDirectory((short) 10); + + short pageSize = (short) (pageDirectory.getEffectivePageSize() - 10); + Page p1 = pageDirectory.getPageWithSpace(pageSize); + p1.unpin(); + + pageDirectory.updateFreeSpace(p1, pageSize); + try { + pageDirectory.updateFreeSpace(p1, (short) 10); + fail(); + } catch (PageException e) { /* do nothing */ } + } + + @Test + public void testUpdateFreeSpaceInvalid2() { + createPageDirectory((short) 10); + + short pageSize = (short) (pageDirectory.getEffectivePageSize() - 10); + Page p1 = pageDirectory.getPageWithSpace(pageSize); + p1.unpin(); + + try { + pageDirectory.updateFreeSpace(p1, (short) - 1); + fail(); + } catch (IllegalArgumentException e) { /* do nothing */ } + + try { + pageDirectory.updateFreeSpace(p1, (short) (pageSize + 1)); + fail(); + } catch (IllegalArgumentException e) { /* do nothing */ } + } + + @Test + public void testIterator() { + createPageDirectory((short) 0); + createPageDirectory((short) (pageDirectory.getEffectivePageSize() - 30)); + + int numRequests = 100; + List pages = new ArrayList<>(); + for (int i = 0; i < numRequests; ++i) { + Page page = pageDirectory.getPageWithSpace((short) 13); + if (pages.size() == 0 || !pages.get(pages.size() - 1).equals(page)) { + pages.add(page); + } + page.unpin(); + } + + Iterator iter = pageDirectory.iterator(); + for (Page page : pages) { + assertTrue(iter.hasNext()); + + Page p = iter.next(); + p.unpin(); + assertEquals(page, p); + } + } + + @Test + public void testIteratorWithDeletes() { + createPageDirectory((short) 0); + createPageDirectory((short) (pageDirectory.getEffectivePageSize() - 30)); + + int numRequests = 100; + List pages = new ArrayList<>(); + for (int i = 0; i < numRequests; ++i) { + Page page = pageDirectory.getPageWithSpace((short) 13); + if (pages.size() == 0 || !pages.get(pages.size() - 1).equals(page)) { + pages.add(page); + } + page.unpin(); + } + + Iterator iterator = pages.iterator(); + while (iterator.hasNext()) { + iterator.next(); + if (iterator.hasNext()) { + pageDirectory.updateFreeSpace(iterator.next(), (short) 30); + iterator.remove(); + } + } + + Iterator iter = pageDirectory.iterator(); + for (Page page : pages) { + assertTrue(iter.hasNext()); + + Page p = iter.next(); + p.unpin(); + assertEquals(page, p); + } + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/table/TestRecord.java b/src/test/java/edu/berkeley/cs186/database/table/TestRecord.java new file mode 100644 index 0000000..a507b22 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/table/TestRecord.java @@ -0,0 +1,70 @@ +package edu.berkeley.cs186.database.table; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import java.util.Arrays; + +import edu.berkeley.cs186.database.categories.*; +import org.junit.Test; + +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.databox.BoolDataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.databox.Type; +import org.junit.experimental.categories.Category; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestRecord { + @Test + public void testToAndFromBytes() { + Schema[] schemas = { + new Schema(Arrays.asList("x"), Arrays.asList(Type.boolType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.intType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.floatType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.stringType(3))), + new Schema(Arrays.asList("w", "x", "y", "z"), + Arrays.asList(Type.boolType(), Type.intType(), + Type.floatType(), Type.stringType(3))), + }; + + Record[] records = { + new Record(Arrays.asList(new BoolDataBox(false))), + new Record(Arrays.asList(new IntDataBox(0))), + new Record(Arrays.asList(new FloatDataBox(0f))), + new Record(Arrays.asList(new StringDataBox("foo", 3))), + new Record(Arrays.asList( + new BoolDataBox(false), + new IntDataBox(0), + new FloatDataBox(0f), + new StringDataBox("foo", 3) + )) + }; + + assert(schemas.length == records.length); + for (int i = 0; i < schemas.length; ++i) { + Schema s = schemas[i]; + Record r = records[i]; + assertEquals(r, Record.fromBytes(ByteBuffer.wrap(r.toBytes(s)), s)); + } + } + + @Test + public void testEquals() { + Record a = new Record(Arrays.asList(new BoolDataBox(false))); + Record b = new Record(Arrays.asList(new BoolDataBox(true))); + Record c = new Record(Arrays.asList(new IntDataBox(0))); + + assertEquals(a, a); + assertNotEquals(a, b); + assertNotEquals(a, c); + assertNotEquals(b, a); + assertEquals(b, b); + assertNotEquals(b, c); + assertNotEquals(c, a); + assertNotEquals(c, b); + assertEquals(c, c); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/table/TestRecordId.java b/src/test/java/edu/berkeley/cs186/database/table/TestRecordId.java new file mode 100644 index 0000000..9038b59 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/table/TestRecordId.java @@ -0,0 +1,67 @@ +package edu.berkeley.cs186.database.table; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.ByteBuffer; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestRecordId { + @Test + public void testSizeInBytes() { + assertEquals(10, RecordId.getSizeInBytes()); + } + + @Test + public void testToAndFromBytes() { + for (int i = 0; i < 10; ++i) { + for (short j = 0; j < 10; ++j) { + RecordId rid = new RecordId(i, j); + assertEquals(rid, RecordId.fromBytes(ByteBuffer.wrap(rid.toBytes()))); + } + } + } + + @Test + public void testEquals() { + RecordId a = new RecordId(0, (short) 0); + RecordId b = new RecordId(1, (short) 0); + RecordId c = new RecordId(0, (short) 1); + + assertEquals(a, a); + assertNotEquals(a, b); + assertNotEquals(a, c); + assertNotEquals(b, a); + assertEquals(b, b); + assertNotEquals(b, c); + assertNotEquals(c, a); + assertNotEquals(c, b); + assertEquals(c, c); + } + + @Test + public void testCompareTo() { + RecordId a = new RecordId(0, (short) 0); + RecordId b = new RecordId(0, (short) 1); + RecordId c = new RecordId(1, (short) 0); + RecordId d = new RecordId(1, (short) 1); + + assertTrue(a.compareTo(a) == 0); + assertTrue(b.compareTo(b) == 0); + assertTrue(c.compareTo(c) == 0); + assertTrue(d.compareTo(d) == 0); + + assertTrue(a.compareTo(b) < 0); + assertTrue(b.compareTo(c) < 0); + assertTrue(c.compareTo(d) < 0); + + assertTrue(d.compareTo(c) > 0); + assertTrue(c.compareTo(b) > 0); + assertTrue(b.compareTo(a) > 0); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/table/TestSchema.java b/src/test/java/edu/berkeley/cs186/database/table/TestSchema.java new file mode 100644 index 0000000..6491baa --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/table/TestSchema.java @@ -0,0 +1,133 @@ +package edu.berkeley.cs186.database.table; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.fail; + +import java.util.Arrays; +import java.util.List; + +import edu.berkeley.cs186.database.categories.*; +import org.junit.Test; + +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.common.ByteBuffer; +import edu.berkeley.cs186.database.databox.BoolDataBox; +import edu.berkeley.cs186.database.databox.DataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.databox.Type; +import org.junit.experimental.categories.Category; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestSchema { + @Test + public void testSizeInBytes() { + Schema[] schemas = { + // Single column. + new Schema(Arrays.asList("x"), Arrays.asList(Type.boolType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.intType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.floatType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.stringType(1))), + new Schema(Arrays.asList("x"), Arrays.asList(Type.stringType(10))), + + // Multiple columns. + new Schema(Arrays.asList("x", "y", "z"), + Arrays.asList(Type.boolType(), Type.intType(), Type.floatType())), + new Schema(Arrays.asList("x", "y"), + Arrays.asList(Type.boolType(), Type.stringType(42))), + }; + + int[] expectedSizes = {1, 4, 4, 1, 10, 9, 43}; + + assert(schemas.length == expectedSizes.length); + for (int i = 0; i < schemas.length; ++i) { + assertEquals(expectedSizes[i], schemas[i].getSizeInBytes()); + } + } + + @Test + public void testVerifyValidRecords() { + try { + Schema[] schemas = { + new Schema(Arrays.asList(), Arrays.asList()), + new Schema(Arrays.asList("x"), Arrays.asList(Type.boolType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.intType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.floatType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.stringType(1))), + new Schema(Arrays.asList("x"), Arrays.asList(Type.stringType(2))), + }; + List> values = Arrays.asList( + Arrays.asList(), + Arrays.asList(new BoolDataBox(false)), + Arrays.asList(new IntDataBox(0)), + Arrays.asList(new FloatDataBox(0f)), + Arrays.asList(new StringDataBox("a", 1)), + Arrays.asList(new StringDataBox("ab", 2)) + ); + + assert(schemas.length == values.size()); + for (int i = 0; i < schemas.length; ++i) { + Schema s = schemas[i]; + List v = values.get(i); + assertEquals(new Record(v), s.verify(v)); + } + } catch (DatabaseException e) { + fail(e.getMessage()); + } + } + + @Test(expected = DatabaseException.class) + public void testVerifyWrongSize() { + Schema schema = new Schema(Arrays.asList("x"), Arrays.asList(Type.boolType())); + List values = Arrays.asList(); + schema.verify(values); + } + + @Test(expected = DatabaseException.class) + public void testVerifyWrongType() { + Schema schema = new Schema(Arrays.asList("x"), Arrays.asList(Type.boolType())); + List values = Arrays.asList(new IntDataBox(42)); + schema.verify(values); + } + + @Test + public void testToAndFromBytes() { + Schema[] schemas = { + // Single column. + new Schema(Arrays.asList("x"), Arrays.asList(Type.boolType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.intType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.floatType())), + new Schema(Arrays.asList("x"), Arrays.asList(Type.stringType(1))), + new Schema(Arrays.asList("x"), Arrays.asList(Type.stringType(10))), + + // Multiple columns. + new Schema(Arrays.asList("x", "y", "z"), + Arrays.asList(Type.boolType(), Type.intType(), Type.floatType())), + new Schema(Arrays.asList("x", "y"), + Arrays.asList(Type.boolType(), Type.stringType(42))), + }; + + for (Schema schema : schemas) { + assertEquals(schema, Schema.fromBytes(ByteBuffer.wrap(schema.toBytes()))); + } + } + + @Test + public void testEquals() { + Schema a = new Schema(Arrays.asList("x"), Arrays.asList(Type.intType())); + Schema b = new Schema(Arrays.asList("y"), Arrays.asList(Type.intType())); + Schema c = new Schema(Arrays.asList("x"), Arrays.asList(Type.boolType())); + + assertEquals(a, a); + assertNotEquals(a, b); + assertNotEquals(a, c); + assertNotEquals(b, a); + assertEquals(b, b); + assertNotEquals(b, c); + assertNotEquals(c, a); + assertNotEquals(c, b); + assertEquals(c, c); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/table/TestTable.java b/src/test/java/edu/berkeley/cs186/database/table/TestTable.java new file mode 100644 index 0000000..787a0b9 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/table/TestTable.java @@ -0,0 +1,449 @@ +package edu.berkeley.cs186.database.table; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.memory.Page; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import edu.berkeley.cs186.database.DatabaseException; +import edu.berkeley.cs186.database.TestUtils; +import edu.berkeley.cs186.database.common.iterator.BacktrackingIterator; +import edu.berkeley.cs186.database.databox.IntDataBox; + +@Category({HW99Tests.class, SystemTests.class}) +public class TestTable { + private static final String TABLENAME = "testtable"; + private MemoryHeapFile heapFile; + private Table table; + private Schema schema; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void beforeEach() { + this.schema = TestUtils.createSchemaWithAllTypes(); + this.heapFile = new MemoryHeapFile(); + this.table = new Table(TABLENAME, schema, heapFile, new DummyLockContext()); + } + + @After + public void cleanup() { + this.heapFile.close(); + } + + private static Record createRecordWithAllTypes(int i) { + Record r = TestUtils.createRecordWithAllTypes(); + r.getValues().set(1, new IntDataBox(i)); + return r; + } + + @Test + public void testGetNumRecordsPerPage() { + assertEquals(10, schema.getSizeInBytes()); + assertEquals(4060, heapFile.getEffectivePageSize()); + // 50 + (400 * 10) = 4050 + // 51 + (408 * 10) = 4131 + assertEquals(400, table.getNumRecordsPerPage()); + } + + @Test + public void testSingleInsertAndGet() { + Record r = createRecordWithAllTypes(0); + RecordId rid = table.addRecord(r.getValues()); + assertEquals(r, table.getRecord(rid)); + } + + @Test + public void testThreePagesOfInserts() { + List rids = new ArrayList<>(); + for (int i = 0; i < table.getNumRecordsPerPage() * 3; ++i) { + Record r = createRecordWithAllTypes(i); + rids.add(table.addRecord(r.getValues())); + } + + for (int i = 0; i < table.getNumRecordsPerPage() * 3; ++i) { + Record r = createRecordWithAllTypes(i); + assertEquals(r, table.getRecord(rids.get(i))); + } + } + + @Test + public void testSingleDelete() { + Record r = createRecordWithAllTypes(0); + RecordId rid = table.addRecord(r.getValues()); + assertEquals(r, table.deleteRecord(rid)); + } + + @Test + public void testThreePagesOfDeletes() { + List rids = new ArrayList<>(); + for (int i = 0; i < table.getNumRecordsPerPage() * 3; ++i) { + Record r = createRecordWithAllTypes(i); + rids.add(table.addRecord(r.getValues())); + } + + for (int i = 0; i < table.getNumRecordsPerPage() * 3; ++i) { + Record r = createRecordWithAllTypes(i); + assertEquals(r, table.deleteRecord(rids.get(i))); + } + } + + @Test(expected = DatabaseException.class) + public void testGetDeletedRecord() { + Record r = createRecordWithAllTypes(0); + RecordId rid = table.addRecord(r.getValues()); + table.deleteRecord(rid); + table.getRecord(rid); + } + + @Test + public void testUpdateSingleRecord() { + Record rOld = createRecordWithAllTypes(0); + Record rNew = createRecordWithAllTypes(42); + + RecordId rid = table.addRecord(rOld.getValues()); + assertEquals(rOld, table.updateRecord(rNew.getValues(), rid)); + assertEquals(rNew, table.getRecord(rid)); + } + + @Test + public void testThreePagesOfUpdates() { + List rids = new ArrayList<>(); + for (int i = 0; i < table.getNumRecordsPerPage() * 3; ++i) { + Record r = createRecordWithAllTypes(i); + rids.add(table.addRecord(r.getValues())); + } + + for (int i = 0; i < table.getNumRecordsPerPage() * 3; ++i) { + Record rOld = createRecordWithAllTypes(i); + Record rNew = createRecordWithAllTypes(i * 10000); + assertEquals(rOld, table.updateRecord(rNew.getValues(), rids.get(i))); + assertEquals(rNew, table.getRecord(rids.get(i))); + } + } + + @Test + public void testReloadTable() { + // We add 42 to make sure we have some incomplete pages. + int numRecords = table.getNumRecordsPerPage() * 2 + 42; + + List rids = new ArrayList<>(); + for (int i = 0; i < numRecords; ++i) { + Record r = createRecordWithAllTypes(i); + rids.add(table.addRecord(r.getValues())); + } + + table = new Table(table.getName(), table.getSchema(), heapFile, new DummyLockContext()); + for (int i = 0; i < numRecords; ++i) { + Record r = createRecordWithAllTypes(i); + assertEquals(r, table.getRecord(rids.get(i))); + } + } + + @Test + public void testReloadTableThenWriteMoreRecords() { + // We add 42 to make sure we have some incomplete pages. + int numRecords = table.getNumRecordsPerPage() * 2 + 42; + + List rids = new ArrayList<>(); + for (int i = 0; i < numRecords; ++i) { + Record r = createRecordWithAllTypes(i); + rids.add(table.addRecord(r.getValues())); + } + + table = new Table(table.getName(), table.getSchema(), heapFile, new DummyLockContext()); + for (int i = numRecords; i < 2 * numRecords; ++i) { + Record r = createRecordWithAllTypes(i); + rids.add(table.addRecord(r.getValues())); + } + + for (int i = 0; i < 2 * numRecords; ++i) { + Record r = createRecordWithAllTypes(i); + assertEquals(r, table.getRecord(rids.get(i))); + } + } + + /** + * Loads some number of pages of records. rids will be loaded with all the record IDs + * of the new records, and the number of records will be returned. + */ + private int setupIteratorTest(List rids, int pages) throws DatabaseException { + int numRecords = table.getNumRecordsPerPage() * pages; + + // Write the records. + for (int i = 0; i < numRecords; ++i) { + Record r = createRecordWithAllTypes(i); + RecordId rid = table.addRecord(r.getValues()); + rids.add(rid); + } + + return numRecords; + } + + /** + * See above; this overload should be used when the list of record IDs is not + * needed. + */ + private int setupIteratorTest(int pages) throws DatabaseException { + List rids = new ArrayList<>(); + return setupIteratorTest(rids, pages); + } + + /** + * Performs a simple loop checking (end - start)/incr records from iter, and + * assuming values of recordWithAllTypes(i), where start <= i < end and + * i increments by incr. + */ + private void checkSequentialRecords(int start, int end, int incr, + BacktrackingIterator iter) { + for (int i = start; i < end; i += incr) { + assertTrue(iter.hasNext()); + assertEquals(createRecordWithAllTypes(i), iter.next()); + } + } + + /** + * Basic test over a full page of records to check that next/hasNext work. + */ + @Test + public void testRIDPageIterator() throws DatabaseException { + int numRecords = setupIteratorTest(1); + Iterator pages = table.pageIterator(); + Page page = pages.next(); + + BacktrackingIterator iter = new RecordIterator(table, table.new RIDPageIterator(page)); + checkSequentialRecords(0, numRecords, 1, iter); + assertFalse(iter.hasNext()); + } + + /** + * Basic test over a half-full page of records, with a missing first/last + * record and gaps between every record, to check that next/hasNext work. + */ + @Test + public void testRIDPageIteratorWithGaps() throws DatabaseException { + List rids = new ArrayList<>(); + int numRecords = setupIteratorTest(rids, 1); + + // Delete every other record and the last record. + for (int i = 0; i < numRecords - 1; i += 2) { + table.deleteRecord(rids.get(i)); + } + table.deleteRecord(rids.get(numRecords - 1)); + + Iterator pages = table.pageIterator(); + Page page = pages.next(); + + BacktrackingIterator iter = new RecordIterator(table, table.new RIDPageIterator(page)); + checkSequentialRecords(1, numRecords - 1, 2, iter); + assertFalse(iter.hasNext()); + } + + /** + * Basic test making sure that RIDPageIterator handles mark/reset properly. + */ + @Test + public void testRIDPageIteratorMarkReset() throws DatabaseException { + int numRecords = setupIteratorTest(1); + Iterator pages = table.pageIterator(); + Page page = pages.next(); + + BacktrackingIterator iter = new RecordIterator(table, table.new RIDPageIterator(page)); + checkSequentialRecords(0, numRecords / 2, 1, iter); + iter.markPrev(); + checkSequentialRecords(numRecords / 2, numRecords, 1, iter); + assertFalse(iter.hasNext()); + iter.reset(); + // -1 because the record before the mark must also be returned + checkSequentialRecords(numRecords / 2 - 1, numRecords, 1, iter); + assertFalse(iter.hasNext()); + + // resetting twice to the same mark should be fine. + iter.reset(); + checkSequentialRecords(numRecords / 2 - 1, numRecords, 1, iter); + assertFalse(iter.hasNext()); + } + + /** + * Extra test making sure that RIDPageIterator handles mark/reset properly. + */ + @Test + public void testRIDPageIteratorMarkResetExtra() throws DatabaseException { + int numRecords = setupIteratorTest(1); + Iterator pages = table.pageIterator(); + Page page = pages.next(); + + BacktrackingIterator iter = new RecordIterator(table, table.new RIDPageIterator(page)); + // This should do nothing. + iter.reset(); + checkSequentialRecords(0, numRecords, 1, iter); + assertFalse(iter.hasNext()); + + page.pin(); + iter = new RecordIterator(table, table.new RIDPageIterator(page)); + // This should also do nothing. + iter.markPrev(); + iter.reset(); + checkSequentialRecords(0, numRecords, 1, iter); + assertFalse(iter.hasNext()); + + // No effective mark = no reset. + iter.reset(); + assertFalse(iter.hasNext()); + + // mark last record + iter.markPrev(); + iter.reset(); + checkSequentialRecords(numRecords - 1, numRecords, 1, iter); + assertFalse(iter.hasNext()); + } + + /** + * Basic test making sure that RIDPageIterator handles mark/reset properly, + * but with gaps between each record. + */ + @Test + public void testRIDPageIteratorMarkResetWithGaps() throws DatabaseException { + List rids = new ArrayList<>(); + int numRecords = setupIteratorTest(rids, 1); + + // Delete every other record and the last record. + for (int i = 0; i < numRecords - 1; i += 2) { + table.deleteRecord(rids.get(i)); + } + table.deleteRecord(rids.get(numRecords - 1)); + + Iterator pages = table.pageIterator(); + Page page = pages.next(); + + BacktrackingIterator iter = new RecordIterator(table, table.new RIDPageIterator(page)); + + int stop = numRecords / 2; + if (stop % 2 == 0) { + ++stop; + } + checkSequentialRecords(1, stop, 2, iter); + iter.markPrev(); + checkSequentialRecords(stop, numRecords - 1, 2, iter); + assertFalse(iter.hasNext()); + iter.reset(); + // -2 because the record before the mark must also be returned + checkSequentialRecords(stop - 2, numRecords - 1, 2, iter); + assertFalse(iter.hasNext()); + + // resetting twice to the same mark should be fine. + iter.reset(); + checkSequentialRecords(stop - 2, numRecords - 1, 2, iter); + assertFalse(iter.hasNext()); + } + + /** + * Extra test making sure that RIDPageIterator handles mark/reset properly, + * but with gaps between each record. + */ + @Test + public void testRIDPageIteratorMarkResetWithGapsExtra() throws DatabaseException { + List rids = new ArrayList<>(); + int numRecords = setupIteratorTest(rids, 1); + + // Delete every other record and the last record. + for (int i = 0; i < numRecords - 1; i += 2) { + table.deleteRecord(rids.get(i)); + } + table.deleteRecord(rids.get(numRecords - 1)); + + Iterator pages = table.pageIterator(); + Page page = pages.next(); + + BacktrackingIterator iter = new RecordIterator(table, table.new RIDPageIterator(page)); + // This should do nothing. + iter.reset(); + checkSequentialRecords(1, numRecords - 1, 2, iter); + assertFalse(iter.hasNext()); + + page.pin(); + iter = new RecordIterator(table, table.new RIDPageIterator(page)); + // This should also do nothing. + iter.markPrev(); + iter.reset(); + checkSequentialRecords(1, numRecords - 1, 2, iter); + assertFalse(iter.hasNext()); + + // No effective mark = no reset. + iter.reset(); + assertFalse(iter.hasNext()); + + // mark last record + iter.markPrev(); + iter.reset(); + // check last record + checkSequentialRecords(numRecords - 3, numRecords - 1, 2, iter); + assertFalse(iter.hasNext()); + } + + /** + * Simple test of TableIterator over three pages of records with no gaps. + */ + @Test + public void testTableIterator() { + // We add 42 to make sure we have some incomplete pages. + int numRecords = table.getNumRecordsPerPage() * 2 + 42; + + // Write the records. + for (int i = 0; i < numRecords; ++i) { + Record r = createRecordWithAllTypes(i); + table.addRecord(r.getValues()); + } + + // Iterate once. + BacktrackingIterator iter = table.iterator(); + checkSequentialRecords(0, numRecords, 1, iter); + assertFalse(iter.hasNext()); + + // Iterate twice for good measure. + iter = table.iterator(); + checkSequentialRecords(0, numRecords, 1, iter); + assertFalse(iter.hasNext()); + } + + /** + * Simple test of TableIterator over three pages of records with every other + * record missing. + */ + @Test + public void testTableIteratorWithGaps() { + // We add 42 to make sure we have some incomplete pages. + int numRecords = table.getNumRecordsPerPage() * 2 + 42; + + // Write the records. + List rids = new ArrayList<>(); + for (int i = 0; i < numRecords; ++i) { + Record r = createRecordWithAllTypes(i); + rids.add(table.addRecord(r.getValues())); + } + + // Delete every other record. + for (int i = 0; i < numRecords; i += 2) { + table.deleteRecord(rids.get(i)); + } + + // Iterate. + BacktrackingIterator iter = table.iterator(); + checkSequentialRecords(1, numRecords, 2, iter); + assertFalse(iter.hasNext()); + } +} diff --git a/src/test/java/edu/berkeley/cs186/database/table/stats/TestHistogram.java b/src/test/java/edu/berkeley/cs186/database/table/stats/TestHistogram.java new file mode 100644 index 0000000..7b98667 --- /dev/null +++ b/src/test/java/edu/berkeley/cs186/database/table/stats/TestHistogram.java @@ -0,0 +1,255 @@ +package edu.berkeley.cs186.database.table.stats; + +import edu.berkeley.cs186.database.TimeoutScaling; +import edu.berkeley.cs186.database.categories.*; +import edu.berkeley.cs186.database.common.PredicateOperator; +import edu.berkeley.cs186.database.concurrency.DummyLockContext; +import edu.berkeley.cs186.database.table.*; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import edu.berkeley.cs186.database.TestUtils; + +import edu.berkeley.cs186.database.databox.IntDataBox; +import edu.berkeley.cs186.database.databox.StringDataBox; +import edu.berkeley.cs186.database.databox.FloatDataBox; +import edu.berkeley.cs186.database.databox.BoolDataBox; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.*; + +@Category({HW3Tests.class, HW3Part2Tests.class}) +public class TestHistogram { + private static final String TABLENAME = "testtable"; + private Table table; + + // 400ms max per method tested. + @Rule + public TestRule globalTimeout = new DisableOnDebug(Timeout.millis((long) ( + 400 * TimeoutScaling.factor))); + + @Before + public void beforeEach() { + Schema schema = TestUtils.createSchemaWithAllTypes(); + HeapFile heapFile = new MemoryHeapFile(); + this.table = new Table(TABLENAME, schema, heapFile, new DummyLockContext()); + } + + //creates a record with all specified types + private static Record createRecordWithAllTypes(boolean a1, int a2, String a3, float a4) { + Record r = TestUtils.createRecordWithAllTypes(); + r.getValues().set(0, new BoolDataBox(a1)); + r.getValues().set(1, new IntDataBox(a2)); + r.getValues().set(2, new StringDataBox(a3, 1)); + r.getValues().set(3, new FloatDataBox(a4)); + return r; + } + + @Test + @Category(PublicTests.class) + public void testBuildHistogramBasic() { + //creates a 101 records int 0 to 100 + for (int i = 0; i < 101; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + table.addRecord(r.getValues()); + } + + //creates a histogram of 10 buckets + Histogram h = new Histogram(10); + h.buildHistogram(table, 1); //build on the integer col + + assertEquals(101, h.getCount()); //count updated properly + + assertEquals(101, h.getNumDistinct()); //distinct count updated properly + + for (int i = 0; i < 9; i++) { + assertEquals(10, h.get(i).getCount()); + } + + assertEquals(11, h.get(9).getCount()); + } + + @Test + @Category(PublicTests.class) + public void testBuildHistogramString() { + //creates a 101 records int 0 to 100 + for (int i = 0; i < 101; ++i) { + Record r = createRecordWithAllTypes(false, 0, "" + (char) i, 0.0f); + table.addRecord(r.getValues()); + } + + //creates a histogram of 10 buckets + Histogram h = new Histogram(10); + h.buildHistogram(table, 2); //build on the integer col + + assertEquals(101, h.getCount()); //count updated properly + + assertEquals(101, h.getNumDistinct()); //distinct count updated properly + } + + @Test + @Category(PublicTests.class) + public void testBuildHistogramEdge() { + //creates a 101 records int 0 to 100 + for (int i = 0; i < 101; ++i) { + Record r = createRecordWithAllTypes(false, 0, "!", 0.0f); + table.addRecord(r.getValues()); + } + + //creates a histogram of 10 buckets + Histogram h = new Histogram(10); + h.buildHistogram(table, 1); //build on the integer col + + assertEquals(101, h.getCount()); //count updated properly + + assertEquals(1, h.getNumDistinct()); //distinct count updated properly + + for (int i = 0; i < 9; i++) { + assertEquals(0, h.get(i).getCount()); + } + + assertEquals(101, h.get(9).getCount()); + } + + @Test + @Category(PublicTests.class) + public void testEquality() { + //creates a 100 records int 0 to 99 + for (int i = 0; i < 100; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + table.addRecord(r.getValues()); + } + + //creates a histogram of 10 buckets + Histogram h = new Histogram(10); + h.buildHistogram(table, 1); //build on the integer col + + //Should return [0.1,0,0,0,0,0,0,0,0,0,0] + float [] result = h.filter(PredicateOperator.EQUALS, new IntDataBox(5)); + assertEquals(0.1, result[0], 0.00001); + for (int i = 1; i < 10; i++) { + assertEquals(0.0, result[i], 0.00001); + } + + //Should return [0.9,1,1,1,1,1,1,1,1,1,1] + result = h.filter(PredicateOperator.NOT_EQUALS, new IntDataBox(5)); + assertEquals(0.9, result[0], 0.00001); + for (int i = 1; i < 10; i++) { + assertEquals(1.0, result[i], 0.00001); + } + + //Should return [0,0,0,0,0,0,0,0,0,0,0.1] + result = h.filter(PredicateOperator.EQUALS, new IntDataBox(99)); + for (int i = 0; i < 9; i++) { + assertEquals(0.0, result[i], 0.00001); + } + assertEquals(0.1, result[9], 0.00001); + + //Should return [0,0,0,0,0,0,0,0,0,0,0.0] + result = h.filter(PredicateOperator.EQUALS, new IntDataBox(100)); + for (int i = 0; i < 10; i++) { + assertEquals(0.0, result[i], 0.00001); + } + + //Should return [0,0,0,0,0,0,0,0,0,0,0.0] + result = h.filter(PredicateOperator.EQUALS, new IntDataBox(-1)); + for (int i = 0; i < 10; i++) { + assertEquals(0.0, result[i], 0.00001); + } + + //Should return [1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0] + result = h.filter(PredicateOperator.NOT_EQUALS, new IntDataBox(-1)); + for (int i = 0; i < 10; i++) { + assertEquals(1.0, result[i], 0.00001); + } + } + + @Test + @Category(PublicTests.class) + public void testGreaterThan() { + //creates a 101 records int 0 to 100 + for (int i = 0; i <= 100; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + table.addRecord(r.getValues()); + } + + //creates a histogram of 10 buckets + Histogram h = new Histogram(10); + h.buildHistogram(table, 1); //build on the integery col + + //Should return [0.1,1,1,1,1,1,1,1,1,1,1] + float [] result = h.filter(PredicateOperator.GREATER_THAN, new IntDataBox(9)); + assertEquals(0.1, result[0], 0.00001); + for (int i = 1; i < 10; i++) { + assertEquals(1.0, result[i], 0.00001); + } + + //Should return [0.0,1,1,1,1,1,1,1,1,1,1] + result = h.filter(PredicateOperator.GREATER_THAN, new IntDataBox(10)); + assertEquals(0.0, result[0], 0.00001); + for (int i = 1; i < 10; i++) { + assertEquals(1.0, result[i], 0.00001); + } + + //Should return [1,1,1,1,1,1,1,1,1,1,1] + result = h.filter(PredicateOperator.GREATER_THAN, new IntDataBox(-1)); + for (int i = 0; i < 10; i++) { + assertEquals(1.0, result[i], 0.00001); + } + + //Should return [0,0,0,0,0,0,0,0,0,0,0.0] + result = h.filter(PredicateOperator.GREATER_THAN, new IntDataBox(101)); + for (int i = 0; i < 10; i++) { + assertEquals(0.0, result[i], 0.00001); + } + } + + @Test + @Category(PublicTests.class) + public void testLessThan() { + //creates a 101 records int 0 to 100 + for (int i = 0; i <= 100; ++i) { + Record r = createRecordWithAllTypes(false, i, "!", 0.0f); + table.addRecord(r.getValues()); + } + + //creates a histogram of 10 buckets + Histogram h = new Histogram(10); + h.buildHistogram(table, 1); //build on the integery col + + //Should return [0.9,0,0,0,0,0,0,0,0,0,0] + float [] result = h.filter(PredicateOperator.LESS_THAN, new IntDataBox(9)); + assertEquals(0.9, result[0], 0.00001); + for (int i = 1; i < 10; i++) { + assertEquals(0.0, result[i], 0.00001); + } + + //Should return [1.0,0,0,0,0,0,0,0,0,0,0] + result = h.filter(PredicateOperator.LESS_THAN, new IntDataBox(10)); + assertEquals(1.0, result[0], 0.00001); + for (int i = 1; i < 10; i++) { + assertEquals(0.0, result[i], 0.00001); + } + + //Should return [1,1,1,1,1,1,1,1,1,1,1] + result = h.filter(PredicateOperator.LESS_THAN, new IntDataBox(101)); + for (int i = 0; i < 10; i++) { + assertEquals(1.0, result[i], 0.00001); + } + + //Should return [0,0,0,0,0,0,0,0,0,0,0.0] + result = h.filter(PredicateOperator.LESS_THAN, new IntDataBox(-1)); + for (int i = 0; i < 10; i++) { + assertEquals(0.0, result[i], 0.00001); + } + } + +} diff --git a/src/test/students.csv b/src/test/students.csv new file mode 100644 index 0000000..d47c308 --- /dev/null +++ b/src/test/students.csv @@ -0,0 +1,200 @@ +1,Augustina Mazzoni,Chemistry,1.005420210172708 +2,Rachel Nugent,Art History,3.872256353606213 +3,Wilburn Lamson,Art History,2.886956910371463 +4,Sharilyn Havens,Biology,2.9571260727339514 +5,Jonah Vanhoy,Chemistry,1.757128473162053 +6,Elijah Pinnix,Business,1.0981805569522254 +7,Darron Bladen,Business,1.0759229784499584 +8,Caprice Verdin,Business,1.8687480647602839 +9,Ramiro Mauriello,Biology,3.671119895688771 +10,Adriene Moeller,CS,2.511224621628675 +11,Keenan Toller,Zoology,2.0095761891916935 +12,Krishna Shiflett,Biology,3.132715066590464 +13,Tijuana Emberton,Business,1.519926413227835 +14,Brenda Notaro,Business,2.4723359821511006 +15,Harley Rhynes,Art History,3.9537853409191803 +16,Joannie Leaver,Business,1.3904102805440584 +17,Leota Mecham,Chemistry,1.7842792468434356 +18,Zella Forth,Business,3.265014134538361 +19,Elden Pecor,Art History,1.725189630557897 +20,Kellye Waller,Biology,3.549029200124181 +21,Jorge Stjames,Chemistry,2.6693979910304773 +22,Lino Mona,Chemistry,3.0235158982574903 +23,Hui Ogata,CS,1.1141306033665095 +24,Suzy Hynson,Chemistry,1.8489339514475187 +25,Patrica Romney,Biology,2.4493228757109695 +26,Vikki Dickinson,Business,2.0164438430296743 +27,Maryanne File,Business,1.80718011606369 +28,Thomas Neese,Biology,2.509270952111554 +29,Yukiko Mitts,Zoology,2.0263705831170165 +30,Malia Maring,Zoology,1.5802372260931592 +31,Phylicia Raphael,CS,3.663960034313729 +32,Anjelica Cortinas,CS,1.5873498632723777 +33,Dierdre Jeffers,CS,3.750047396982882 +34,Janelle Wireman,Zoology,1.1683512393715647 +35,Tracey Herdon,CS,2.6027620276199865 +36,Shaneka Mannings,Business,1.362897151527899 +37,Frieda Baize,Business,1.2015557716783654 +38,Fumiko Tsao,Art History,3.127481525747611 +39,Pansy Mumaw,Biology,2.0379136590364046 +40,Senaida Rutherford,Chemistry,1.4132778263831318 +41,Tessa Lipscomb,Zoology,3.980650875715811 +42,Abel Henningsen,Biology,1.956925561862458 +43,Mina Tiedeman,CS,3.7629902969401794 +44,Maryjane Tippie,Business,2.1591182312108295 +45,Deadra Mulhall,Art History,3.5089762682364305 +46,Yevette Lanoue,Business,2.8830213815651122 +47,Kenya Reeve,Zoology,1.1912302669817527 +48,Chin Lindon,Chemistry,3.441029036700732 +49,Federico Holley,Business,2.113072417690852 +50,Florrie Mattie,Zoology,2.7814934226269017 +51,Eugenio Jacobus,Biology,1.4428809443186754 +52,Toshiko Vanwagenen,Art History,2.1387345746520303 +53,Susanne Haugland,Zoology,3.3137135936332642 +54,Joannie Everman,Business,3.7609962062645317 +55,Johnny Bliss,Business,1.9686421162550254 +56,Ione Schuck,Chemistry,2.3369857817742132 +57,Lauren Riemann,Business,2.31167086135883 +58,Shawnee Merle,Business,2.4319340650736225 +59,Cathie Kilduff,Chemistry,3.479132908238493 +60,Aron Veach,Business,3.4528583927823 +61,Mable Faux,Art History,3.500354461633917 +62,Lakeesha Shew,CS,3.3956461331174586 +63,Jacquiline Soder,Chemistry,1.1217747033745442 +64,Gavin Carley,Chemistry,3.665337948428652 +65,Kimberley Grazier,Chemistry,1.607617018265236 +66,Bradford Raffield,Biology,3.262817033281434 +67,Domonique Fernald,CS,1.226340331580115 +68,Francene Chenail,Chemistry,1.4574112493314246 +69,Keiko Sherer,Zoology,3.26035983169056 +70,Leopoldo Hoekstra,Chemistry,2.955967087798275 +71,Abby Wilmeth,Business,2.1280911075537907 +72,Ophelia Paules,Business,1.9187812077168966 +73,Maren Coombe,Business,3.0202261409881137 +74,Young Pine,Business,3.5738091075883593 +75,Marva Venegas,CS,2.0158610815017783 +76,Dianne Underdown,Zoology,1.569729137308229 +77,Tijuana Miranda,Zoology,2.2306690592916594 +78,Solange Godley,Art History,2.056882817169261 +79,Kareen Klemme,Chemistry,3.605818437551586 +80,Mckenzie Krogh,Zoology,3.615648967769329 +81,Benjamin Cooter,Art History,3.006321167717683 +82,Jacinda Crumbley,Art History,1.561227372385432 +83,Eli Dillahunty,Art History,2.8505374968398174 +84,Maude Mcclintock,Zoology,1.935996082606244 +85,Noriko Neilson,Zoology,3.1178359672805076 +86,Lorena Maynor,CS,3.9255997412863244 +87,Mari Moultry,Art History,3.533880402333662 +88,Loni Straughan,CS,1.3106762079967593 +89,Shiloh Gable,Zoology,3.277365216516241 +90,Pinkie Castleman,Biology,3.976364623970541 +91,Cleta Gast,CS,1.0347560732136325 +92,Raquel Beason,Chemistry,1.5485334747991533 +93,Cameron Riggenbach,Business,1.8096741573969957 +94,Erminia Selfridge,CS,2.5332871920084825 +95,Elois Kyler,Chemistry,2.67582677795827 +96,Carson Kingery,Biology,3.399376612065245 +97,Isela Rivers,Zoology,1.1725424100482318 +98,Natacha Burchfield,Art History,2.941274064997769 +99,Ursula Armstrong,Biology,2.190513137542306 +100,Junior Halderman,Chemistry,1.4156586247487972 +101,Sibyl Pinnell,Business,3.443426546750993 +102,Donnell Stelle,CS,3.1076183500117085 +103,Chelsea Holderman,Art History,3.0893242848159956 +104,Nicole Amos,Zoology,2.199267560975421 +105,Jerrica Nowacki,Zoology,3.354563061020292 +106,Taunya Boddy,Art History,1.4282549174382344 +107,Florine Emigh,Zoology,2.101815544958494 +108,Corene Barkley,Art History,3.548484599923123 +109,Taylor Caggiano,Zoology,3.257408083426325 +110,Lashell Castelli,Business,2.5023599094832236 +111,Georgann Lax,CS,3.667080707847653 +112,Renate Manson,Business,1.5608868610525548 +113,Carlyn Lanser,CS,1.6895447179107057 +114,Beckie Vidaurri,Biology,1.1371815835381986 +115,Karolyn Branscum,CS,2.9283235498420406 +116,Elfreda Enfinger,Business,2.471069184859428 +117,Onie Parker,Business,3.2167107231063556 +118,Eugenie Moreau,Chemistry,2.0589164276197347 +119,Selma Nibert,Business,3.4786675783132104 +120,Starr Overson,Art History,2.773724964759487 +121,Yoshie Glueck,Chemistry,3.5093921196828064 +122,Rigoberto Keegan,Biology,1.3346799265166047 +123,Bianca Kirchner,Business,3.251528689128719 +124,Alita Faulkner,Zoology,2.0647624591716895 +125,Dia Hosch,Chemistry,1.4940877000854007 +126,Carlota Sergeant,Chemistry,2.43811472141723 +127,Lizzie Eisert,Chemistry,1.211964439022407 +128,Charlie Kisner,Art History,2.7562745971588845 +129,Sheryll Angevine,Art History,2.7996307121550137 +130,Ivan Swaney,Business,2.250886803698565 +131,Tanner Berndt,Art History,3.9211488674305635 +132,Almeta Bisceglia,Biology,1.0564758304416297 +133,Von Kimberly,Business,3.4724789756919985 +134,Kassandra Lauderback,Business,2.948091249965167 +135,Bobbie Atkins,Business,3.07600986019198 +136,Theola Moates,Zoology,3.571994920981793 +137,Normand Abernathy,Zoology,1.8739601623550368 +138,Hanna Kuiper,Chemistry,2.4710137025581136 +139,Ingeborg Appling,CS,1.808538137875484 +140,Williams Royer,Biology,2.133217915282454 +141,Vannesa Delgiudice,Zoology,2.0042654121276375 +142,Maximina Lombard,CS,1.3655362590477806 +143,Aleta Vandine,Biology,2.675354514412378 +144,Grover Hoggatt,Zoology,3.0602615998339706 +145,Arlyne Mccaulley,Zoology,2.886748456496785 +146,Halina Tinner,Art History,3.690549060628541 +147,Jayson Frankum,Art History,3.791576509800353 +148,Daniella Weingarten,Zoology,1.4551314267167865 +149,Justin Bain,Biology,1.9057009751145935 +150,Sarita Voigt,Biology,2.7827870498865677 +151,Darren Bartolomeo,Chemistry,3.731273043555529 +152,Melodi Hinerman,Business,3.032679105389611 +153,Kina Herrada,Biology,2.272396577530438 +154,Dalene Coachman,CS,2.5128426635276857 +155,Peggie Lenoir,Business,2.398568157853456 +156,Alane Crawshaw,Art History,1.801003654479133 +157,Aurora Terry,Zoology,3.3564203402768396 +158,Angila Romine,Chemistry,1.8492064179334262 +159,Reinaldo Bjorklund,Art History,1.8264880086475288 +160,Nikita Mccrum,Biology,1.3047318153857783 +161,Mikki Sroka,Chemistry,1.0102279874316729 +162,George Marshburn,Zoology,2.8472217875500796 +163,Theda Frye,CS,1.1143589032894488 +164,Ivonne Rudd,CS,2.565551265903017 +165,Corrin Francois,Chemistry,2.9075940422314304 +166,Joaquin June,Biology,3.3945678678794966 +167,Diego Durand,Zoology,3.369793535166474 +168,Grover Navarre,Art History,1.9345595099139348 +169,Bok Artist,Zoology,3.370670492459216 +170,Roberto Blazier,CS,1.4993471648642085 +171,Isadora Gavin,Zoology,1.3247776094970245 +172,Christene Mccloud,Business,1.9554346250471144 +173,Leticia Metcalf,CS,1.6756784309503145 +174,Hank Number,Zoology,1.0490619034894548 +175,Coretta Stefano,Art History,3.7332752903301785 +176,Hiedi Sisler,Business,3.149263709123936 +177,Donald Mckinney,Art History,3.115447382089699 +178,Lamar Menges,Chemistry,2.780094757854773 +179,Alfredo Madden,Business,3.6646488733965716 +180,Tanesha Wiley,Business,2.3803783350232077 +181,Jaunita Presswood,CS,3.7457808001514845 +182,Henry Porco,Art History,1.1531257653658091 +183,Richard Lindholm,Biology,2.0002739640402067 +184,Roxanne Masek,CS,2.3149818175767782 +185,Helene Macleod,CS,2.5925009947479047 +186,Elizabet Trowell,Biology,2.906090426924333 +187,Kasi Chesney,Zoology,1.107816161048091 +188,Carlita Holmon,Art History,1.9484717368141058 +189,Salome Newport,Chemistry,1.7454779935250886 +190,Azucena Galles,Business,1.2519184085958166 +191,Kizzie Cliff,Business,3.913792929559814 +192,Gregg Ahlgren,Business,1.690390383587132 +193,Carolin Keels,CS,2.9916220069580395 +194,Catalina Echavarria,Zoology,3.3984181844218915 +195,Denice Pothier,Chemistry,3.388576246359147 +196,Shauna Dutta,Zoology,2.2527986472805157 +197,Marisela Cue,Business,2.0716201591707373 +198,Hannelore Goya,Art History,1.647341862916107 +199,Ozella Backstrom,Chemistry,3.3695989272830484 +200,Meggan Gasser,Business,1.568863708664711 diff --git a/students.csv b/students.csv new file mode 100644 index 0000000..d47c308 --- /dev/null +++ b/students.csv @@ -0,0 +1,200 @@ +1,Augustina Mazzoni,Chemistry,1.005420210172708 +2,Rachel Nugent,Art History,3.872256353606213 +3,Wilburn Lamson,Art History,2.886956910371463 +4,Sharilyn Havens,Biology,2.9571260727339514 +5,Jonah Vanhoy,Chemistry,1.757128473162053 +6,Elijah Pinnix,Business,1.0981805569522254 +7,Darron Bladen,Business,1.0759229784499584 +8,Caprice Verdin,Business,1.8687480647602839 +9,Ramiro Mauriello,Biology,3.671119895688771 +10,Adriene Moeller,CS,2.511224621628675 +11,Keenan Toller,Zoology,2.0095761891916935 +12,Krishna Shiflett,Biology,3.132715066590464 +13,Tijuana Emberton,Business,1.519926413227835 +14,Brenda Notaro,Business,2.4723359821511006 +15,Harley Rhynes,Art History,3.9537853409191803 +16,Joannie Leaver,Business,1.3904102805440584 +17,Leota Mecham,Chemistry,1.7842792468434356 +18,Zella Forth,Business,3.265014134538361 +19,Elden Pecor,Art History,1.725189630557897 +20,Kellye Waller,Biology,3.549029200124181 +21,Jorge Stjames,Chemistry,2.6693979910304773 +22,Lino Mona,Chemistry,3.0235158982574903 +23,Hui Ogata,CS,1.1141306033665095 +24,Suzy Hynson,Chemistry,1.8489339514475187 +25,Patrica Romney,Biology,2.4493228757109695 +26,Vikki Dickinson,Business,2.0164438430296743 +27,Maryanne File,Business,1.80718011606369 +28,Thomas Neese,Biology,2.509270952111554 +29,Yukiko Mitts,Zoology,2.0263705831170165 +30,Malia Maring,Zoology,1.5802372260931592 +31,Phylicia Raphael,CS,3.663960034313729 +32,Anjelica Cortinas,CS,1.5873498632723777 +33,Dierdre Jeffers,CS,3.750047396982882 +34,Janelle Wireman,Zoology,1.1683512393715647 +35,Tracey Herdon,CS,2.6027620276199865 +36,Shaneka Mannings,Business,1.362897151527899 +37,Frieda Baize,Business,1.2015557716783654 +38,Fumiko Tsao,Art History,3.127481525747611 +39,Pansy Mumaw,Biology,2.0379136590364046 +40,Senaida Rutherford,Chemistry,1.4132778263831318 +41,Tessa Lipscomb,Zoology,3.980650875715811 +42,Abel Henningsen,Biology,1.956925561862458 +43,Mina Tiedeman,CS,3.7629902969401794 +44,Maryjane Tippie,Business,2.1591182312108295 +45,Deadra Mulhall,Art History,3.5089762682364305 +46,Yevette Lanoue,Business,2.8830213815651122 +47,Kenya Reeve,Zoology,1.1912302669817527 +48,Chin Lindon,Chemistry,3.441029036700732 +49,Federico Holley,Business,2.113072417690852 +50,Florrie Mattie,Zoology,2.7814934226269017 +51,Eugenio Jacobus,Biology,1.4428809443186754 +52,Toshiko Vanwagenen,Art History,2.1387345746520303 +53,Susanne Haugland,Zoology,3.3137135936332642 +54,Joannie Everman,Business,3.7609962062645317 +55,Johnny Bliss,Business,1.9686421162550254 +56,Ione Schuck,Chemistry,2.3369857817742132 +57,Lauren Riemann,Business,2.31167086135883 +58,Shawnee Merle,Business,2.4319340650736225 +59,Cathie Kilduff,Chemistry,3.479132908238493 +60,Aron Veach,Business,3.4528583927823 +61,Mable Faux,Art History,3.500354461633917 +62,Lakeesha Shew,CS,3.3956461331174586 +63,Jacquiline Soder,Chemistry,1.1217747033745442 +64,Gavin Carley,Chemistry,3.665337948428652 +65,Kimberley Grazier,Chemistry,1.607617018265236 +66,Bradford Raffield,Biology,3.262817033281434 +67,Domonique Fernald,CS,1.226340331580115 +68,Francene Chenail,Chemistry,1.4574112493314246 +69,Keiko Sherer,Zoology,3.26035983169056 +70,Leopoldo Hoekstra,Chemistry,2.955967087798275 +71,Abby Wilmeth,Business,2.1280911075537907 +72,Ophelia Paules,Business,1.9187812077168966 +73,Maren Coombe,Business,3.0202261409881137 +74,Young Pine,Business,3.5738091075883593 +75,Marva Venegas,CS,2.0158610815017783 +76,Dianne Underdown,Zoology,1.569729137308229 +77,Tijuana Miranda,Zoology,2.2306690592916594 +78,Solange Godley,Art History,2.056882817169261 +79,Kareen Klemme,Chemistry,3.605818437551586 +80,Mckenzie Krogh,Zoology,3.615648967769329 +81,Benjamin Cooter,Art History,3.006321167717683 +82,Jacinda Crumbley,Art History,1.561227372385432 +83,Eli Dillahunty,Art History,2.8505374968398174 +84,Maude Mcclintock,Zoology,1.935996082606244 +85,Noriko Neilson,Zoology,3.1178359672805076 +86,Lorena Maynor,CS,3.9255997412863244 +87,Mari Moultry,Art History,3.533880402333662 +88,Loni Straughan,CS,1.3106762079967593 +89,Shiloh Gable,Zoology,3.277365216516241 +90,Pinkie Castleman,Biology,3.976364623970541 +91,Cleta Gast,CS,1.0347560732136325 +92,Raquel Beason,Chemistry,1.5485334747991533 +93,Cameron Riggenbach,Business,1.8096741573969957 +94,Erminia Selfridge,CS,2.5332871920084825 +95,Elois Kyler,Chemistry,2.67582677795827 +96,Carson Kingery,Biology,3.399376612065245 +97,Isela Rivers,Zoology,1.1725424100482318 +98,Natacha Burchfield,Art History,2.941274064997769 +99,Ursula Armstrong,Biology,2.190513137542306 +100,Junior Halderman,Chemistry,1.4156586247487972 +101,Sibyl Pinnell,Business,3.443426546750993 +102,Donnell Stelle,CS,3.1076183500117085 +103,Chelsea Holderman,Art History,3.0893242848159956 +104,Nicole Amos,Zoology,2.199267560975421 +105,Jerrica Nowacki,Zoology,3.354563061020292 +106,Taunya Boddy,Art History,1.4282549174382344 +107,Florine Emigh,Zoology,2.101815544958494 +108,Corene Barkley,Art History,3.548484599923123 +109,Taylor Caggiano,Zoology,3.257408083426325 +110,Lashell Castelli,Business,2.5023599094832236 +111,Georgann Lax,CS,3.667080707847653 +112,Renate Manson,Business,1.5608868610525548 +113,Carlyn Lanser,CS,1.6895447179107057 +114,Beckie Vidaurri,Biology,1.1371815835381986 +115,Karolyn Branscum,CS,2.9283235498420406 +116,Elfreda Enfinger,Business,2.471069184859428 +117,Onie Parker,Business,3.2167107231063556 +118,Eugenie Moreau,Chemistry,2.0589164276197347 +119,Selma Nibert,Business,3.4786675783132104 +120,Starr Overson,Art History,2.773724964759487 +121,Yoshie Glueck,Chemistry,3.5093921196828064 +122,Rigoberto Keegan,Biology,1.3346799265166047 +123,Bianca Kirchner,Business,3.251528689128719 +124,Alita Faulkner,Zoology,2.0647624591716895 +125,Dia Hosch,Chemistry,1.4940877000854007 +126,Carlota Sergeant,Chemistry,2.43811472141723 +127,Lizzie Eisert,Chemistry,1.211964439022407 +128,Charlie Kisner,Art History,2.7562745971588845 +129,Sheryll Angevine,Art History,2.7996307121550137 +130,Ivan Swaney,Business,2.250886803698565 +131,Tanner Berndt,Art History,3.9211488674305635 +132,Almeta Bisceglia,Biology,1.0564758304416297 +133,Von Kimberly,Business,3.4724789756919985 +134,Kassandra Lauderback,Business,2.948091249965167 +135,Bobbie Atkins,Business,3.07600986019198 +136,Theola Moates,Zoology,3.571994920981793 +137,Normand Abernathy,Zoology,1.8739601623550368 +138,Hanna Kuiper,Chemistry,2.4710137025581136 +139,Ingeborg Appling,CS,1.808538137875484 +140,Williams Royer,Biology,2.133217915282454 +141,Vannesa Delgiudice,Zoology,2.0042654121276375 +142,Maximina Lombard,CS,1.3655362590477806 +143,Aleta Vandine,Biology,2.675354514412378 +144,Grover Hoggatt,Zoology,3.0602615998339706 +145,Arlyne Mccaulley,Zoology,2.886748456496785 +146,Halina Tinner,Art History,3.690549060628541 +147,Jayson Frankum,Art History,3.791576509800353 +148,Daniella Weingarten,Zoology,1.4551314267167865 +149,Justin Bain,Biology,1.9057009751145935 +150,Sarita Voigt,Biology,2.7827870498865677 +151,Darren Bartolomeo,Chemistry,3.731273043555529 +152,Melodi Hinerman,Business,3.032679105389611 +153,Kina Herrada,Biology,2.272396577530438 +154,Dalene Coachman,CS,2.5128426635276857 +155,Peggie Lenoir,Business,2.398568157853456 +156,Alane Crawshaw,Art History,1.801003654479133 +157,Aurora Terry,Zoology,3.3564203402768396 +158,Angila Romine,Chemistry,1.8492064179334262 +159,Reinaldo Bjorklund,Art History,1.8264880086475288 +160,Nikita Mccrum,Biology,1.3047318153857783 +161,Mikki Sroka,Chemistry,1.0102279874316729 +162,George Marshburn,Zoology,2.8472217875500796 +163,Theda Frye,CS,1.1143589032894488 +164,Ivonne Rudd,CS,2.565551265903017 +165,Corrin Francois,Chemistry,2.9075940422314304 +166,Joaquin June,Biology,3.3945678678794966 +167,Diego Durand,Zoology,3.369793535166474 +168,Grover Navarre,Art History,1.9345595099139348 +169,Bok Artist,Zoology,3.370670492459216 +170,Roberto Blazier,CS,1.4993471648642085 +171,Isadora Gavin,Zoology,1.3247776094970245 +172,Christene Mccloud,Business,1.9554346250471144 +173,Leticia Metcalf,CS,1.6756784309503145 +174,Hank Number,Zoology,1.0490619034894548 +175,Coretta Stefano,Art History,3.7332752903301785 +176,Hiedi Sisler,Business,3.149263709123936 +177,Donald Mckinney,Art History,3.115447382089699 +178,Lamar Menges,Chemistry,2.780094757854773 +179,Alfredo Madden,Business,3.6646488733965716 +180,Tanesha Wiley,Business,2.3803783350232077 +181,Jaunita Presswood,CS,3.7457808001514845 +182,Henry Porco,Art History,1.1531257653658091 +183,Richard Lindholm,Biology,2.0002739640402067 +184,Roxanne Masek,CS,2.3149818175767782 +185,Helene Macleod,CS,2.5925009947479047 +186,Elizabet Trowell,Biology,2.906090426924333 +187,Kasi Chesney,Zoology,1.107816161048091 +188,Carlita Holmon,Art History,1.9484717368141058 +189,Salome Newport,Chemistry,1.7454779935250886 +190,Azucena Galles,Business,1.2519184085958166 +191,Kizzie Cliff,Business,3.913792929559814 +192,Gregg Ahlgren,Business,1.690390383587132 +193,Carolin Keels,CS,2.9916220069580395 +194,Catalina Echavarria,Zoology,3.3984181844218915 +195,Denice Pothier,Chemistry,3.388576246359147 +196,Shauna Dutta,Zoology,2.2527986472805157 +197,Marisela Cue,Business,2.0716201591707373 +198,Hannelore Goya,Art History,1.647341862916107 +199,Ozella Backstrom,Chemistry,3.3695989272830484 +200,Meggan Gasser,Business,1.568863708664711 diff --git a/turn_in.py b/turn_in.py new file mode 100644 index 0000000..f5196a1 --- /dev/null +++ b/turn_in.py @@ -0,0 +1,204 @@ +import argparse +import json +import os +import re +import shutil +import tempfile +import subprocess + +HW_LIST = ['hw0', 'hw1', 'hw2', 'hw3_part1', 'hw3_part2', 'hw4_part1', 'hw4_part2', 'hw5'] + +def check_student_id(student_id): + m = re.match(r'[0-9]{8,10}', student_id) + if not m or len(student_id) not in (8, 10): + print('Error: Please double check that your student id is entered correctly. It should only include digits 0-9 and be of length 8 or 10.') + exit() + return student_id + +def test_category(assignment): + return assignment[2:].replace('_p', 'P').replace('Part2', '') + +def files_to_copy(assignment): + files = { + 'hw0': ['src/main/java/edu/berkeley/cs186/database/databox/StringDataBox.java'], + 'hw1': ['hw1.sql'], + 'hw2': [ + 'src/main/java/edu/berkeley/cs186/database/index/BPlusTree.java', + 'src/main/java/edu/berkeley/cs186/database/index/BPlusNode.java', + 'src/main/java/edu/berkeley/cs186/database/index/InnerNode.java', + 'src/main/java/edu/berkeley/cs186/database/index/LeafNode.java', + ], + 'hw3_part1': [ + 'src/main/java/edu/berkeley/cs186/database/query/BNLJOperator.java', + 'src/main/java/edu/berkeley/cs186/database/query/SortOperator.java', + 'src/main/java/edu/berkeley/cs186/database/query/SortMergeOperator.java', + ], + 'hw3_part2': [ + 'src/main/java/edu/berkeley/cs186/database/query/BNLJOperator.java', + 'src/main/java/edu/berkeley/cs186/database/query/SortOperator.java', + 'src/main/java/edu/berkeley/cs186/database/query/SortMergeOperator.java', + 'src/main/java/edu/berkeley/cs186/database/query/QueryPlan.java', + 'src/main/java/edu/berkeley/cs186/database/table/stats/Histogram.java', + ], + 'hw4_part1': [ + 'src/main/java/edu/berkeley/cs186/database/concurrency/LockType.java', + 'src/main/java/edu/berkeley/cs186/database/concurrency/LockManager.java', + 'src/main/java/edu/berkeley/cs186/database/concurrency/LockContext.java', + ], + 'hw4_part2': [ + 'src/main/java/edu/berkeley/cs186/database/concurrency/LockType.java', + 'src/main/java/edu/berkeley/cs186/database/concurrency/LockManager.java', + 'src/main/java/edu/berkeley/cs186/database/concurrency/LockContext.java', + 'src/main/java/edu/berkeley/cs186/database/concurrency/LockUtil.java', + 'src/main/java/edu/berkeley/cs186/database/index/LeafNode.java', + 'src/main/java/edu/berkeley/cs186/database/index/InnerNode.java', + 'src/main/java/edu/berkeley/cs186/database/index/BPlusTree.java', + 'src/main/java/edu/berkeley/cs186/database/memory/Page.java', + 'src/main/java/edu/berkeley/cs186/database/table/PageDirectory.java', + 'src/main/java/edu/berkeley/cs186/database/table/Table.java', + 'src/main/java/edu/berkeley/cs186/database/Database.java', + ], + 'hw5': [ + 'src/main/java/edu/berkeley/cs186/database/recovery/ARIESRecoveryManager.java', + 'src/test/java/edu/berkeley/cs186/database/recovery/TestARIESStudent.java', + ], + } + return files[assignment] + +def get_path(hw_file): + index = hw_file.rfind('/') + if index == -1: + return '' + return hw_file[:index] + +def get_dirs(hw_files): + dirs = set() + for hw in hw_files: + dirs.add(get_path(hw)) + return dirs + +def create_hw_dirs(tempdir, assignment, dirs): + for d in dirs: + try: + tmp_hw_path = tempdir + '/' + assignment + '/' + d + if not os.path.isdir(tmp_hw_path): + os.makedirs(tmp_hw_path) + except OSError: + print('Error: Creating directory %s failed' % tmp_hw_path) + exit() + return tempdir + '/' + assignment + +def copy_file(filename, hw_path, tmp_hw_path): + student_file_path = hw_path + '/' + filename + tmp_student_file_path = tmp_hw_path + '/' + get_path(filename) + if not os.path.isfile(student_file_path): + print('Error: could not find file at %s' % student_file_path) + exit() + shutil.copy(student_file_path, tmp_student_file_path) + +def create_submission_gpg(student_id, tmp_hw_path): + # Create submission_info.txt with student id info + data = {'student_id': student_id} + txt_submission_path = tmp_hw_path + '/submission_info.txt' + with open(txt_submission_path, 'w+') as outfile: + json.dump(data, outfile) + + # Encrypt submission_info.txt to submission_info.gpg + # and delete submission_info.txt + public_key_file = os.getcwd() + '/public.key' + if not os.path.isfile(public_key_file): + print('Error: Missing the public.key file') + exit() + + import_cmd = ['gpg', '-q', '--import', 'public.key'] + import_run = subprocess.run(import_cmd) + import_run.check_returncode() + + gpg_submission_path = tmp_hw_path + '/submission_info.gpg' + encrypt_cmd = ['gpg', '--output', gpg_submission_path, '--trust-model', 'always', '-e', '-q', '-r', 'CS186 Staff', txt_submission_path] + encrypt_run = subprocess.run(encrypt_cmd) + encrypt_run.check_returncode() + + os.remove(txt_submission_path) + +def compile_submission(tmp_hw_path, hw_files, assign): + old_cwd = os.getcwd() + with tempfile.TemporaryDirectory() as tempdir: + os.chdir(tempdir) + + r = subprocess.run(['git', 'init', '-q']) + r.check_returncode() + + r = subprocess.run(['git', 'remote', 'add', 'local', 'file://' + old_cwd]) + r.check_returncode() + + r = subprocess.run(['git', 'pull', 'local', 'origin/master', '-q']) + r.check_returncode() + + for filename in hw_files: + copy_file(filename, tmp_hw_path, tempdir) + + if assign != 'hw1': + print('Compiling submission...') + r = subprocess.run(['mvn', 'clean', 'compile', '-q', '-B'], stdout=subprocess.PIPE) + + if r.returncode != 0: + print('\nError: compilation failed with status', r.returncode, '\n') + # last 7 lines are not useful output + print('\n'.join(r.stdout.decode('utf-8').split('\n')[:-7]), '\n') + + os.chdir(old_cwd) + exit() + + print('Running public tests...') + r = subprocess.run(['mvn', 'test', '-q', '-B', '-DHW=' + test_category(assign), '-Ppublic', '-DgenerateReports=false', '-Dsurefire.printSummary=false'], stdout=subprocess.PIPE) + + if r.returncode != 0: + print('\nWarning: some test failures\n') + # last 12 lines are not useful output + print('\n'.join(r.stdout.decode('utf-8').split('\n')[:-12]), '\n') + else: + print('Running public tests...') + r = subprocess.run(['./test.sh'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if r.returncode != 0: + print('\nWarning: some test failures\n') + print('\n'.join(r.stdout.decode('utf-8').split('\n')), '\n') + + os.chdir(old_cwd) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='hw submission script') + parser.add_argument('--student-id', type=check_student_id, help='Berkeley student ID') + parser.add_argument('--assignment', help='assignment number', choices=HW_LIST) + parser.add_argument('--skip-compile', action='store_true', help='use to skip compilation step') + args = parser.parse_args() + + if not args.student_id: + args.student_id = input('Please enter your Berkeley student ID: ') + check_student_id(args.student_id) + + if not args.assignment: + args.assignment = input('Please enter the assignment number (one of {}): '.format(str(HW_LIST))) + if args.assignment not in HW_LIST: + print('Error: please make sure you entered a valid assignment number') + exit() + + with tempfile.TemporaryDirectory() as tempdir: + hw_files = files_to_copy(args.assignment) + dirs = get_dirs(hw_files) + tmp_hw_path = create_hw_dirs(tempdir, args.assignment, dirs) + for filename in hw_files: + copy_file(filename, os.getcwd(), tmp_hw_path) + + if not args.skip_compile: + compile_submission(tmp_hw_path, hw_files, args.assignment) + + create_submission_gpg(args.student_id, tmp_hw_path) + + # Create zip file + hw_zip_path = os.getcwd() + '/' + args.assignment + '.zip' + shutil.make_archive(args.assignment, 'zip', tempdir) + + print('Created ' + args.assignment + '.zip') +