diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000..416e174d --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,10 @@ +changelog: + categories: + - title: New Features + labels: [enhancement, feature] + - title: Bug Fixes + labels: [bug, fix] + - title: Documentation + labels: [documentation] + - title: Other Changes + labels: ["*"] diff --git a/.github/workflows/NuGetPublish.yml b/.github/workflows/NuGetPublish.yml new file mode 100644 index 00000000..32d9444c --- /dev/null +++ b/.github/workflows/NuGetPublish.yml @@ -0,0 +1,37 @@ +name: Publish to NuGet.org + +on: + release: + types: [published] + +jobs: + publish: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 8.0.x + 9.0.x + 10.0.x + + - name: Extract version from tag + id: version + shell: bash + run: echo "VERSION=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT + + - name: Build + run: dotnet build Numerics/Numerics.csproj -c Release -p:Version=${{ steps.version.outputs.VERSION }} + + - name: Test (all frameworks) + run: dotnet test -c Release --no-build + + - name: Pack + run: dotnet pack Numerics/Numerics.csproj -c Release --no-build -p:Version=${{ steps.version.outputs.VERSION }} + + - name: Push to NuGet.org + run: dotnet nuget push "Numerics/bin/Release/RMC.Numerics.${{ steps.version.outputs.VERSION }}.nupkg" --api-key ${{ secrets.NUGET_ORG_API_KEY }} --source https://api.nuget.org/v3/index.json --skip-duplicate diff --git a/.gitignore b/.gitignore index b34cbaa7..0049b80c 100644 --- a/.gitignore +++ b/.gitignore @@ -370,3 +370,4 @@ MigrationBackup/ #Numerics Specific /TestResults +/.claude/settings.local.json diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..a562fff1 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,45 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +type: software +title: "Numerics" +version: "2.0.0" +date-released: "2026-03-08" +license: BSD-3-Clause +repository-code: "https://github.com/USACE-RMC/Numerics" +url: "https://github.com/USACE-RMC/Numerics" +abstract: >- + Numerics is a free and open-source .NET library providing numerical methods, + probability distributions, statistical analysis, and Bayesian inference tools + for quantitative risk assessment in water resources engineering. +keywords: + - statistics + - probability distributions + - MCMC + - hydrology + - flood frequency analysis + - copulas + - L-moments + - optimization + - .NET + - C# +authors: + - family-names: "Smith" + given-names: "C. Haden" + email: "cole.h.smith@usace.army.mil" + affiliation: "U.S. Army Corps of Engineers, Risk Management Center" + orcid: "https://orcid.org/0000-0002-4651-9890" + - family-names: "Fields" + given-names: "Woodrow L." + affiliation: "U.S. Army Corps of Engineers, Risk Management Center" + - family-names: "Gonzalez" + given-names: "Julian" + affiliation: "U.S. Army Corps of Engineers, Risk Management Center" + - family-names: "Niblett" + given-names: "Sadie" + affiliation: "U.S. Army Corps of Engineers, Risk Management Center" + - family-names: "Beam" + given-names: "Brennan" + affiliation: "U.S. Army Corps of Engineers, Hydrologic Engineering Center" + - family-names: "Skahill" + given-names: "Brian" + affiliation: "Fariborz Maseeh Department of Mathematics and Statistics, Portland State University" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..780f67a3 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,13 @@ +# Code of Conduct + +This project follows the [Contributor Covenant Code of Conduct v2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). + +Please read the full text at the link above. In summary, we are committed to providing a welcoming and inclusive environment for everyone. Be respectful, constructive, and professional in all interactions. + +## Reporting + +If you experience or witness unacceptable behavior, please contact the project maintainers at cole.h.smith@usace.army.mil. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.1. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..1d5e4099 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,63 @@ +# Contributing to Numerics + +Thank you for your interest in contributing to Numerics! We welcome bug reports, feature requests, validation results, and other feedback from the community. + +## Review Capacity + +Numerics is maintained by a small team within the U.S. Army Corps of Engineers Risk Management Center (USACE-RMC). **Our capacity to review external pull requests is very limited.** We prioritize issues and bug reports, which are always welcome and will be reviewed as resources permit. + +If you plan to submit a pull request, please open an issue first to discuss your proposed change. This helps avoid duplicated effort and ensures your contribution aligns with the project's direction. + +## How to Contribute + +### Report a Bug + +If you find a bug, please [open an issue](../../issues/new) and include: + +- Steps to reproduce the problem +- Input data and configuration (if applicable) +- Expected behavior vs. actual behavior +- Software version and operating system +- Any relevant error messages or screenshots + +### Request a Feature + +Feature requests are welcome. Please [open an issue](../../issues/new) describing: + +- The use case or problem you are trying to solve +- How you envision the feature working +- Any references to statistical methods or published literature + +### Report Validation Results + +Given the life-safety applications of this software, independent validation is especially valuable. If you have compared Numerics results against other software (e.g., R packages, published tables, or analytical solutions), we would appreciate hearing about it through an issue. + +### Submit a Pull Request + +Pull requests may take several weeks or longer to review. Before submitting code: + +1. **Open an issue first** to discuss the proposed change +2. **Follow the coding standards**, including XML documentation on all public types and members +3. **Include unit tests** that validate against known results (R, SciPy, Mathematica, or published tables) +4. **Use MSTest** (`[TestClass]`, `[TestMethod]`) as the testing framework +5. **Ensure a clean build** with zero errors and zero warnings + +## Developer Certificate of Origin + +By submitting a pull request, you certify under the [Developer Certificate of Origin (DCO) Version 1.1](https://developercertificate.org/) that you have the right to submit the work under the license associated with this project and that you agree to the DCO. + +All contributions will be released under the same license as the project (see [LICENSE](LICENSE)). + +## Federal Government Contributors + +U.S. Federal law prevents the government from accepting gratuitous services unless certain conditions are met. By submitting a pull request, you acknowledge that your services are offered without expectation of payment and that you expressly waive any future pay claims against the U.S. Federal government related to your contribution. + +If you are a U.S. Federal government employee and use a `*.mil` or `*.gov` email address, your contribution is understood to have been created in whole or in part as part of your official duties and is not subject to domestic copyright protection under 17 USC 105. + +## Security + +If you discover a security vulnerability, please do **not** open a public issue. Instead, contact the RMC team directly through official USACE-RMC channels. + +## License + +See [LICENSE](LICENSE) for details. This software is provided by USACE-RMC under a BSD-style license with a no-endorsement clause. diff --git a/JOSS_TODO.md b/JOSS_TODO.md new file mode 100644 index 00000000..f6c8cecf --- /dev/null +++ b/JOSS_TODO.md @@ -0,0 +1,219 @@ +# JOSS Submission Checklist + +Pre-submission checklist for the Journal of Open Source Software (JOSS). +JOSS review criteria: https://joss.readthedocs.io/en/latest/review_criteria.html +JOSS paper format: https://joss.readthedocs.io/en/latest/paper.html (750-1,750 words) + +--- + +## Section A: JOSS Requirements Compliance + +### Software Requirements + +| # | Requirement | Status | Evidence | +|---|-------------|--------|----------| +| 1 | Open source license (OSI-approved) | MET | BSD-3-Clause in `LICENSE`; declared in `CITATION.cff` | +| 2 | Version control (public repository) | MET | https://github.com/USACE-RMC/Numerics | +| 3 | README with project description | MET | `README.md` with description, install, docs, badges | +| 4 | Installation instructions | MET | NuGet install in `README.md` and `docs/getting-started.md` | +| 5 | Usage examples | MET | `docs/getting-started.md`, `docs/index.md` Quick Start, paper Example section | +| 6 | API documentation | MET | 25+ Markdown files in `docs/`; XML documentation generated from source | +| 7 | Community guidelines (CONTRIBUTING) | MET | `CONTRIBUTING.md` with bug reports, PRs, DCO, security policy | +| 8 | Code of Conduct | MET | `CODE_OF_CONDUCT.md` (Contributor Covenant v2.1) | +| 9 | Automated tests | MET | 1,006 MSTest methods across 150 test classes in `Test_Numerics/` | +| 10 | Continuous integration | MET | 3 GitHub Actions workflows (`Integration.yml`, `Snapshot.yml`, `Release.yml`) | +| 11 | Substantial scholarly effort | MET | 60,000+ LOC library, 34,000+ LOC tests, 248 source files | +| 12 | Sustained development history | MET | 2.5 years (Sep 2023 - Mar 2026), 270+ commits, 7+ contributors | +| 13 | Statement of need | MET | Paper includes Statement of Need section | +| 14 | State of the field / related work | MET | Paper includes State of the Field section | +| 15 | Zero or documented dependencies | MET | Zero runtime dependencies for .NET 8+; polyfills only for .NET Framework 4.8.1 | +| 16 | Functionality matches claims | MET | All claims verified: 43 distributions, 8 MCMC samplers (RWMH, ARWMH, DEMCz, DEMCzs, HMC, NUTS, Gibbs, SNIS), 5+ optimizers, copulas, ML, bootstrap | +| 17 | Software installable | REMAINING | Publish `RMC.Numerics` to nuget.org (see Step 4 below) | + +### Paper Requirements + +| # | Requirement | Status | Evidence | +|---|-------------|--------|----------| +| 18 | Paper in `paper/paper.md` | MET | `paper/paper.md` with YAML frontmatter | +| 19 | Bibliography in `paper/paper.bib` | MET | `paper/paper.bib` with 29 references | +| 20 | Summary section | MET | Lines 39-41 | +| 21 | Statement of Need section | MET | Lines 43-54 | +| 22 | State of the Field section | MET | Lines 56-62 | +| 23 | Research impact / ongoing projects | MET | Lines 64-75 (6 production applications) | +| 24 | Acknowledgements | MET | Lines 119-121 | +| 25 | AI Usage Disclosure | MET | Lines 115-117 | +| 26 | Word count 750-1,750 | MET | ~1,215 words (within range) | +| 27 | Submitting author ORCID | MET | `0000-0002-4651-9890` for C. Haden Smith | +| 28 | Author affiliations | MET | 3 affiliations listed | +| 29 | All references have DOIs where available | MET | 21 DOIs + 7 URLs for web-only resources | +| 30 | Code example in paper | MET | Lines 85-113 (bootstrap uncertainty analysis) | + +### Repository Metadata + +| # | Requirement | Status | Evidence | +|---|-------------|--------|----------| +| 31 | CITATION.cff | MET | `CITATION.cff` (CFF v1.2.0, BSD-3-Clause) | +| 32 | codemeta.json | MET | `codemeta.json` created | +| 33 | Version tag matching submission | REMAINING | Need to create `v2.0.0` tag after merge to `main` | +| 34 | Archived release with DOI | REMAINING | Need Zenodo archive | + +--- + +## Section B: BES Paper Review Comments (Addressed) + +| # | Comment | Status | Action | +|---|---------|--------|--------| +| 1 | Co-author ORCIDs (Fields, Gonzalez, Niblett, Beam) | DONE | All ORCIDs added to paper.md | +| 2 | "Differential Evolution MCMC" → "Adaptive Differential Evolution MCMC" (line 53) | DONE | Clarifies DE-MCz vs DE-MC | +| 3 | "needed for reliable calibration" → "commonly applied for Bayesian inference" (line 61) | DONE | More accurate characterization | +| 4 | Gonzalez: Software Design choppy language | DONE | Rewrote section with accurate base class descriptions, removed First/Second/Third structure | +| 5 | Gonzalez: Redundant "zero dependencies" in Summary | DONE | Removed from Summary, kept in Software Design where it is explained | +| 6 | Gonzalez: Why is CI important? | DONE | Added "preventing regressions in numerical accuracy" to CI sentence | +| 7 | Gonzalez: References not in markdown | N/A | JOSS uses Pandoc + paper.bib to render citations at build time | +| 8 | MCMC sampler count (was "six", should be "eight") | DONE | Updated Summary and Statement of Need to include DEMCzs and SNIS | + +--- + +## Section C: Pre-Release Verification + +- [ ] Run full test suite across all target frameworks: + ``` + dotnet test --framework net8.0 + dotnet test --framework net9.0 + dotnet test --framework net10.0 + dotnet test --framework net481 + ``` +- [ ] Confirm all 1,006+ tests pass on every target +- [ ] Build the NuGet package locally: + ``` + dotnet pack Numerics/Numerics.csproj -c Release + ``` +- [ ] Verify the .nupkg contains assemblies for all 4 TFMs + +--- + +## Section D: Release Roadmap (Step-by-Step) + +### Step 1: Create Pull Request to `main` +- [ ] Push latest `bugfixes-and-enhancements` to origin +- [ ] Create PR: + ``` + gh pr create --base main --head bugfixes-and-enhancements \ + --title "v2.0.0: Major update" --body-file RELEASE_NOTES.md + ``` +- [ ] Wait for CI (Integration.yml) to pass +- [ ] Review and merge PR + +### Step 2: Tag and GitHub Release +- [ ] After merge: + ``` + git checkout main && git pull + git tag v2.0.0 + git push origin v2.0.0 + ``` +- [ ] Create GitHub Release: + ``` + gh release create v2.0.0 --title "v2.0.0 — Major Update" --notes-file RELEASE_NOTES.md + ``` +- [ ] `Release.yml` auto-triggers → pushes to internal USACE Nexus +- [ ] `NuGetPublish.yml` auto-triggers → pushes to nuget.org + +### Step 3: One-Time NuGet.org Setup (before first release) +- [ ] Create/verify nuget.org account at https://www.nuget.org/ +- [ ] Generate API key at https://www.nuget.org/account/apikeys: + - Name: `GitHub Actions - Numerics` + - Expiration: 365 days + - Glob pattern: `RMC.Numerics` + - Scopes: "Push new packages and package versions" +- [ ] Add secret to GitHub repo at https://github.com/USACE-RMC/Numerics/settings/secrets/actions: + - Name: `NUGET_ORG_API_KEY` + - Value: the API key from above + +### Step 4: Verify NuGet Package +- [ ] Check https://www.nuget.org/packages/RMC.Numerics/ (may take 10-15 min to index) +- [ ] Test installation: + ``` + dotnet new console -o TestInstall && cd TestInstall + dotnet add package RMC.Numerics --version 2.0.0 + dotnet build + ``` + +### Step 5: Zenodo Archival +- [ ] Go to https://zenodo.org and log in with GitHub +- [ ] Enable the `USACE-RMC/Numerics` repository in Zenodo's GitHub integration +- [ ] Zenodo will automatically archive the GitHub Release and mint a DOI +- [ ] Copy the Zenodo DOI badge and add it to `README.md` +- [ ] Update `CITATION.cff` with the Zenodo DOI if desired + +### Step 6: Submit to JOSS +- [ ] Go to https://joss.theoj.org/papers/new +- [ ] Enter the repository URL: `https://github.com/USACE-RMC/Numerics` +- [ ] Enter the Zenodo archive DOI +- [ ] Confirm software version matches the tagged release +- [ ] Submit the paper + +### Step 7: Post-Submission +- [ ] Add the JOSS status badge to `README.md` once the review issue is created: + ```markdown + [![status](https://joss.theoj.org/papers//status.svg)](https://joss.theoj.org/papers/) + ``` +- [ ] Monitor the review issue for editor/reviewer feedback +- [ ] Address any review comments promptly (JOSS reviews typically take 4-8 weeks) + +--- + +## Section E: v2.0.0 Release Notes + +### v2.0.0 — Major Update + +This is a major update to Numerics with 274 files changed, 24,476 insertions, and 4,400 deletions since v1.0.0. Highlights include new distributions, improved MCMC inference, enhanced numerical methods, and comprehensive documentation. + +#### New Distributions +- Dirichlet distribution (multivariate) +- Multinomial distribution (multivariate) +- Multivariate Student-t distribution +- Student-t copula + +#### Bayesian Inference & MCMC +- Improved Gelman-Rubin convergence diagnostics +- Refactored Noncentral-T to use Brent.Solve +- Enhanced MCMC sampler reliability and convergence + +#### Numerical Methods +- Linear algebra enhancements +- Root-finding improvements +- ODE solver improvements +- Improved adaptive integration + +#### Data & Statistics +- Time series download improvements +- Hypothesis test enhancements +- Enhanced parameter estimation methods +- Autocorrelation and convergence diagnostics improvements + +#### Optimization +- Comprehensive correctness improvements across all optimizers + +#### Machine Learning +- Documentation consolidation +- Code quality improvements + +#### Infrastructure & Documentation +- Added .NET 10.0 target framework (now targets net8.0, net9.0, net10.0, net481) +- Zero runtime dependencies maintained +- 1,006+ unit tests validated against published references +- JOSS paper and metadata (CITATION.cff, codemeta.json) +- CONTRIBUTING.md and CODE_OF_CONDUCT.md +- 25+ documentation files covering all library capabilities +- NuGet publishing workflow for nuget.org + +--- + +## Notes + +- **Word limit**: JOSS papers should be 750-1,750 words (https://joss.readthedocs.io/en/latest/paper.html). Current paper is ~1,215 words. +- **"Jery R. Stedinger"**: This spelling in `paper.bib` is correct (confirmed on USGS publications). Be prepared to explain if a reviewer questions it. +- **England 2019 vs 2018**: Bulletin 17C was originally published March 2018; the `2019` date refers to the v1.1 revision. Both are acceptable in the literature. +- **ter Braak citation**: The 2008 paper describes DE-MCzs (with snooker updater). The library has both `DEMCz` and `DEMCzs` classes. The citation is appropriate since the 2008 paper supersedes the 2006 original. +- **NuGet API key expiration**: The nuget.org API key expires after 365 days max. Set a calendar reminder to regenerate it. +- **Versioning**: Version is derived from git tags (e.g., `v2.1.0` → package version `2.1.0`). The `.csproj` version is a fallback for local builds only. diff --git a/Numerics.sln b/Numerics.sln index 065265bc..3d423960 100644 --- a/Numerics.sln +++ b/Numerics.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.9.34723.18 +# Visual Studio Version 18 +VisualStudioVersion = 18.2.11430.68 d18.0 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Numerics", "Numerics\Numerics.csproj", "{9C8B75DE-ABF2-42D3-96C7-18405AD16488}" EndProject diff --git a/Numerics/Data/Interpolation/Bilinear.cs b/Numerics/Data/Interpolation/Bilinear.cs index 9d7e0b18..178ac713 100644 --- a/Numerics/Data/Interpolation/Bilinear.cs +++ b/Numerics/Data/Interpolation/Bilinear.cs @@ -119,7 +119,7 @@ public bool UseSmartSearch { _useSmartSearch = value; X1LI.UseSmartSearch = value; - X1LI.UseSmartSearch = value; + X2LI.UseSmartSearch = value; } } diff --git a/Numerics/Data/Interpolation/CubicSpline.cs b/Numerics/Data/Interpolation/CubicSpline.cs index 6a32a3b1..84114b05 100644 --- a/Numerics/Data/Interpolation/CubicSpline.cs +++ b/Numerics/Data/Interpolation/CubicSpline.cs @@ -28,6 +28,9 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +using System; +using System.Collections.Generic; + namespace Numerics.Data { /// @@ -62,7 +65,7 @@ public class CubicSpline : Interpolater { /// - /// Construct new linear interpolation. + /// Construct new cubic spline interpolation. /// /// List of x-values. /// List of y-values. @@ -75,7 +78,7 @@ public CubicSpline(IList xValues, IList yValues, SortOrder sortO /// /// Stores the array of second derivatives. /// - private double[] y2 = Array.Empty(); + private double[] y2 = null!; /// /// Auxiliary routine to set the second derivatives. If you make changes to the x- or y-values, then you need to call this routine afterwards. diff --git a/Numerics/Data/Interpolation/Linear.cs b/Numerics/Data/Interpolation/Linear.cs index 61ad74e1..92e87f29 100644 --- a/Numerics/Data/Interpolation/Linear.cs +++ b/Numerics/Data/Interpolation/Linear.cs @@ -206,7 +206,7 @@ public double Extrapolate(double x) // Extrapolate // Check for division by zero - if ((y2 - y1) == 0) + if ((x2 - x1) == 0) { y = y1; } diff --git a/Numerics/Data/Interpolation/Support/Search.cs b/Numerics/Data/Interpolation/Support/Search.cs index a4dfaf1f..f460dc7e 100644 --- a/Numerics/Data/Interpolation/Support/Search.cs +++ b/Numerics/Data/Interpolation/Support/Search.cs @@ -762,7 +762,7 @@ public static int Hunt(double xValue, OrderedPairedData orderedPairedData, int s int XLO = start; int XHI; int XM; - var ASCND = default(bool); + var ASCND = orderedPairedData.OrderX == SortOrder.Ascending; int INC; // validate @@ -816,7 +816,7 @@ public static int Hunt(double xValue, OrderedPairedData orderedPairedData, int s XLO = start; // Perform the hunt search algorithm - if (XLO <= 0 | XLO > N) + if (XLO <= 0 || XLO > N) { // The input guess is not useful. // Go immediately to bisection. @@ -849,7 +849,6 @@ public static int Hunt(double xValue, OrderedPairedData orderedPairedData, int s else { // Hunt down - XHI = XLO; XHI = XLO - 1; while (X < orderedPairedData[XLO].X == ASCND) { @@ -965,7 +964,7 @@ public static int Hunt(double xValue, IList ordinateData, int start = XLO = start; // Perform the hunt search algorithm - if (XLO <= 0 | XLO > N) + if (XLO <= 0 || XLO > N) { // The input guess is not useful. // Go immediately to bisection. @@ -998,7 +997,6 @@ public static int Hunt(double xValue, IList ordinateData, int start = else { // Hunt down - XHI = XLO; XHI = XLO - 1; while (X < ordinateData[XLO].X == ASCND) { diff --git a/Numerics/Data/Paired Data/OrderedPairedData.cs b/Numerics/Data/Paired Data/OrderedPairedData.cs index 8632cda5..80eb0eef 100644 --- a/Numerics/Data/Paired Data/OrderedPairedData.cs +++ b/Numerics/Data/Paired Data/OrderedPairedData.cs @@ -35,7 +35,6 @@ using System.Data; using System.Linq; using System.Xml.Linq; -using System.Xml.Serialization; using Numerics.Distributions; namespace Numerics.Data @@ -103,11 +102,23 @@ public class OrderedPairedData : IList, INotifyCollectionChanged private bool _strictY; private SortOrder _orderX; private SortOrder _orderY; - private readonly List _ordinates; + private List _ordinates; /// public event NotifyCollectionChangedEventHandler? CollectionChanged; + /// + /// Gets or sets a value indicating whether collection changed events are suppressed. + /// + /// + /// + /// Set this to true before performing a batch of mutations and then + /// call when the batch is complete. + /// This avoids firing an event for every individual mutation. + /// + /// + public bool SuppressCollectionChanged { get; set; } = false; + /// /// Represents if the paired dataset has valid ordinates and order. /// @@ -304,6 +315,22 @@ public OrderedPairedData(XElement el) #region Methods + /// + /// Raises a event + /// unconditionally, regardless of the flag. + /// + /// + /// + /// Call this after a bulk operation once + /// has been set back to false (or while still true if desired) + /// to notify listeners that the collection has changed. + /// + /// + public void RaiseCollectionChangedReset() + { + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); + } + /// /// Get or set the ordinate at a specified index. /// @@ -326,14 +353,11 @@ public Ordinate this[int index] { if (OrdinateValid(index) == true) { Validate(); } } - // - //We might need to add this check if performance suffers - //if (SupressCollectionChanged == false) - //{ - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, value, oldValue)); - //} + if (SuppressCollectionChanged == false) + { + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, value, oldValue, index)); + } } - } } @@ -432,7 +456,8 @@ public bool Remove(Ordinate item) if (itemIndex == -1) return false; _ordinates.RemoveAt(itemIndex); Validate(); - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, itemIndex)); + if (SuppressCollectionChanged == false) + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, itemIndex)); return true; } @@ -446,7 +471,8 @@ public void RemoveAt(int index) var item = _ordinates[index]; _ordinates.RemoveAt(index); Validate(); - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index)); + if (SuppressCollectionChanged == false) + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index)); } /// @@ -461,7 +487,8 @@ public void RemoveRange(int index, int count) for (int i = index; i < count; i++) { items.Add(_ordinates[i]); } _ordinates.RemoveRange(index, count); Validate(); - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, items, index)); + if (SuppressCollectionChanged == false) + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, items, index)); } /// @@ -472,7 +499,8 @@ public void Add(Ordinate item) { _ordinates.Add(item); IsValid = OrdinateValid(_ordinates.Count - 1); - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item, _ordinates.Count - 1)); + if (SuppressCollectionChanged == false) + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item, _ordinates.Count - 1)); } /// @@ -485,7 +513,8 @@ public void Insert(int index, Ordinate item) _ordinates.Insert(index, item); // only need to set valid state if it is true. if it is already false then inserting can't make it true. if (IsValid) IsValid = OrdinateValid(index); - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item, index)); + if (SuppressCollectionChanged == false) + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item, index)); } /// @@ -493,9 +522,17 @@ public void Insert(int index, Ordinate item) /// public void Clear() { + if (_ordinates.Count == 0) + return; + + bool wasSuppressed = SuppressCollectionChanged; + SuppressCollectionChanged = true; _ordinates.Clear(); + SuppressCollectionChanged = wasSuppressed; + IsValid = true; - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); + if (SuppressCollectionChanged == false) + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); } /// @@ -1437,14 +1474,15 @@ private double TriangleArea(Ordinate point1, Ordinate point2, Ordinate point3) /// and number of points in the search region. public OrderedPairedData LangSimplify(double tolerance, int lookAhead) { - if (lookAhead <= 1 | tolerance <= 0) { return this; } + if (_ordinates == null || lookAhead <= 1 || tolerance <= 0) + return this; List ordinates = new List(); int count = _ordinates.Count; int offset; - if (lookAhead > count - 1) { lookAhead = count - 1; } - + if (lookAhead > count - 1) + lookAhead = count - 1; ordinates.Add(_ordinates[0]); for (int i = 0; i < count; i++) diff --git a/Numerics/Data/Paired Data/Ordinate.cs b/Numerics/Data/Paired Data/Ordinate.cs index 153787ae..cb538a3c 100644 --- a/Numerics/Data/Paired Data/Ordinate.cs +++ b/Numerics/Data/Paired Data/Ordinate.cs @@ -68,7 +68,7 @@ public Ordinate(double xValue, double yValue) X = xValue; Y = yValue; IsValid = true; - if (double.IsInfinity(X) | double.IsNaN(X) || double.IsInfinity(Y) | double.IsNaN(Y)) { IsValid = false; } + if (double.IsInfinity(X) || double.IsNaN(X) || double.IsInfinity(Y) || double.IsNaN(Y)) { IsValid = false; } } diff --git a/Numerics/Data/Paired Data/ProbabilityOrdinate.cs b/Numerics/Data/Paired Data/ProbabilityOrdinate.cs index 7d7dee7b..df1e6707 100644 --- a/Numerics/Data/Paired Data/ProbabilityOrdinate.cs +++ b/Numerics/Data/Paired Data/ProbabilityOrdinate.cs @@ -328,6 +328,34 @@ private void AddDefaults() return (isValid, messages); } + /// + /// Gets or sets a value indicating whether collection changed events are suppressed. + /// + /// + /// + /// Set this to true before performing a batch of mutations and then + /// call when the batch is complete. + /// This avoids firing an event for every individual mutation. + /// + /// + public bool SuppressCollectionChanged { get; set; } = false; + + /// + /// Raises a event + /// unconditionally, regardless of the flag. + /// + /// + /// + /// Call this after a bulk operation once + /// has been set back to false (or while still true if desired) + /// to notify listeners that the collection has changed. + /// + /// + public void RaiseCollectionChangedReset() + { + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); + } + #region Mutating methods with change notifications /// @@ -487,12 +515,24 @@ private void AddDefaults() /// /// Removes all elements from the list and raises the appropriate change events. /// + /// + /// + /// If is false, a single + /// event is raised after + /// all items are removed. If suppress is already true, no event is + /// raised (the caller is expected to raise Reset when the batch is complete). + /// + /// public new void Clear() { if (Count == 0) return; + bool wasSuppressed = SuppressCollectionChanged; + SuppressCollectionChanged = true; base.Clear(); + SuppressCollectionChanged = wasSuppressed; + OnPropertyChanged(nameof(Count)); OnPropertyChanged("Item[]"); OnCollectionChanged( @@ -542,13 +582,17 @@ protected virtual void OnPropertyChanged(string propertyName) } /// - /// Raises the event with the specified arguments. + /// Raises the event with the specified arguments, + /// unless is true. /// /// /// that describes the change. /// protected virtual void OnCollectionChanged(NotifyCollectionChangedEventArgs e) { + if (SuppressCollectionChanged) + return; + var handler = CollectionChanged; if (handler != null) handler(this, e); diff --git a/Numerics/Data/Paired Data/UncertainOrderedPairedData.cs b/Numerics/Data/Paired Data/UncertainOrderedPairedData.cs index 4d9ab6a2..05fbb4c5 100644 --- a/Numerics/Data/Paired Data/UncertainOrderedPairedData.cs +++ b/Numerics/Data/Paired Data/UncertainOrderedPairedData.cs @@ -32,7 +32,6 @@ using System.Collections; using System.Collections.Generic; using System.Collections.Specialized; -using System.Data; using System.Globalization; using System.Linq; using System.Xml.Linq; @@ -167,7 +166,7 @@ public SortOrder OrderY /// /// Boolean of whether or not to invoke anything on CollectionChanged /// - public bool SupressCollectionChanged { get; set; } = false; + public bool SuppressCollectionChanged { get; set; } = false; /// /// Handles the event of CollectionChanged @@ -245,11 +244,12 @@ public UncertainOrderedPairedData(IList data, bool strictOnX, _uncertainOrdinates = new List(data.Count); for (int i = 0; i < data.Count; i++) { - var o = data[i]; - UnivariateDistributionBase? yValue = o.Y?.Clone(); - if (yValue is not null) { _uncertainOrdinates.Add(new UncertainOrdinate(o.X, yValue)); } + var y = data[i].Y; + if (y is not null) + _uncertainOrdinates.Add(new UncertainOrdinate(data[i].X, y.Clone())); + else + _uncertainOrdinates.Add(data[i]); } - Validate(); } @@ -273,11 +273,13 @@ private UncertainOrderedPairedData(IList data, bool strictOnX _uncertainOrdinates = new List(data.Count); for (int i = 0; i < data.Count; i++) { - var o = data[i]; - UnivariateDistributionBase? yValue = o.Y?.Clone(); - if (yValue is not null) { _uncertainOrdinates.Add(new UncertainOrdinate(o.X, yValue)); } + var y = data[i].Y; + if (y is not null) + _uncertainOrdinates.Add(new UncertainOrdinate(data[i].X, y.Clone())); + else + _uncertainOrdinates.Add(data[i]); } - + _isValid = dataValid; } @@ -287,20 +289,20 @@ private UncertainOrderedPairedData(IList data, bool strictOnX /// The XElement the UncertainOrderPairedData object is being created from. public UncertainOrderedPairedData(XElement el) { - var strictX = el.Attribute("X_Strict"); // Get Order - if (strictX != null) { bool.TryParse(strictX.Value, out _strictX); } - - var strictY = el.Attribute("Y_Strict"); - if (strictY != null) { bool.TryParse(strictY.Value, out _strictY); } - + var xStrictAttr = el.Attribute("X_Strict"); + if (xStrictAttr != null) + bool.TryParse(xStrictAttr.Value, out _strictX); + var yStrictAttr = el.Attribute("Y_Strict"); + if (yStrictAttr != null) + bool.TryParse(yStrictAttr.Value, out _strictY); // Get Strictness - var orderX = el.Attribute("X_Order"); - if (orderX != null) { Enum.TryParse(orderX.Value, out _orderX); } - - var orderY = el.Attribute("Y_Order"); - if (orderY != null) { Enum.TryParse(orderY.Value, out _orderY); } - + var xOrderAttr = el.Attribute("X_Order"); + if (xOrderAttr != null) + Enum.TryParse(xOrderAttr.Value, out _orderX); + var yOrderAttr = el.Attribute("Y_Order"); + if (yOrderAttr != null) + Enum.TryParse(yOrderAttr.Value, out _orderY); // Distribution type Distribution = UnivariateDistributionType.Deterministic; var distributionAttr = el.Attribute("Distribution"); @@ -311,20 +313,19 @@ public UncertainOrderedPairedData(XElement el) Distribution = argresult; } // new prop - var allowDiffAtr = el.Attribute(nameof(AllowDifferentDistributionTypes)); - if (allowDiffAtr != null) + + var allowDiffAttr = el.Attribute(nameof(AllowDifferentDistributionTypes)); + if (allowDiffAttr != null) { - bool.TryParse(allowDiffAtr.Value, out _allowDifferentDistributionTypes); + bool.TryParse(allowDiffAttr.Value, out _allowDifferentDistributionTypes); // Get Ordinates var curveEl = el.Element("Ordinates"); _uncertainOrdinates = new List(); - if (curveEl != null) { foreach (XElement ord in curveEl.Elements(nameof(UncertainOrdinate))) _uncertainOrdinates.Add(new UncertainOrdinate(ord)); } - } else { @@ -336,18 +337,18 @@ public UncertainOrderedPairedData(XElement el) foreach (XElement o in curveEl.Elements("Ordinate")) { var xAttr = o.Attribute("X"); - if ( xAttr != null && double.TryParse(xAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var xout)) { xData.Add(xout); } - else { xData.Add(0.0); } - + double xout = 0d; + if (xAttr != null) + double.TryParse(xAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out xout); + xData.Add(xout); var dist = UnivariateDistributionFactory.CreateDistribution(Distribution); var props = dist.GetParameterPropertyNames; var paramVals = new double[(props.Count())]; - for (int i = 0; i < props.Count(); i++) { - var pAttr = o.Attribute(props[i]); - if ( pAttr != null && double.TryParse(pAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var result)) { paramVals[i] = result; } - else { paramVals[i] = 0.0; } + var propAttr = o.Attribute(props[i]); + if (propAttr != null) + double.TryParse(propAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out paramVals[i]); } dist.SetParameters(paramVals); @@ -418,7 +419,7 @@ public UncertainOrderedPairedData Clone() /// public void Validate() { - if (SupressCollectionChanged == true) return; + if (SuppressCollectionChanged == true) return; _isValid = true; // innocent until proven guilty for (int i = 0; i < _uncertainOrdinates.Count; i++) { @@ -510,16 +511,7 @@ public List GetErrors() return false; for (int i = 0, loopTo = left.Count - 1; i <= loopTo; i++) { - if (left._uncertainOrdinates[i].X != right._uncertainOrdinates[i].X) - return false; - - var leftY = left._uncertainOrdinates[i].Y; - var rightY = right._uncertainOrdinates[i].Y; - if (leftY is null && rightY is null) - continue; - if (leftY is null || rightY is null) - return false; - if (!leftY.Equals(rightY)) + if (left._uncertainOrdinates[i] != right._uncertainOrdinates[i]) return false; } return true; @@ -592,9 +584,9 @@ public UncertainOrdinate this[int index] { if (OrdinateValid(index) == true) { Validate(); } } - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) { - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, value, oldvalue)); + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, value, oldvalue, index)); } } } @@ -622,7 +614,7 @@ public bool Remove(UncertainOrdinate item) return false; _uncertainOrdinates.RemoveAt(itemIndex); Validate(); - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, itemIndex)); return true; } @@ -636,7 +628,7 @@ public void RemoveAt(int index) var itemRemoved = _uncertainOrdinates[index]; _uncertainOrdinates.RemoveAt(index); Validate(); - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, itemRemoved, index)); } @@ -657,7 +649,7 @@ public void RemoveRange(int index, int count) _uncertainOrdinates.RemoveRange(index, count); Validate(); - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, itemsRemoved, index)); } @@ -682,7 +674,7 @@ public void RemoveRange(int[] rowIndicesToRemove) _uncertainOrdinates.RemoveAt(sortedIndices[i]); } Validate(); - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) { if (isContinuous) { @@ -704,7 +696,7 @@ public void Add(UncertainOrdinate item) _uncertainOrdinates.Add(item); if (OrdinateValid(_uncertainOrdinates.Count - 1) == false) _isValid = false; - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item, _uncertainOrdinates.Count - 1)); } @@ -723,7 +715,7 @@ public void AddRange(IList items) if (OrdinateValid(_uncertainOrdinates.Count - 1) == false) _isValid = false; } - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, items.ToList(), startIndex)); } @@ -742,7 +734,7 @@ public void Insert(int index, UncertainOrdinate item) _isValid = false; } - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item, index)); } @@ -764,7 +756,7 @@ public void InsertRange(int index, IList items) _isValid = false; } } - if (SupressCollectionChanged == false) + if (SuppressCollectionChanged == false) CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, items.ToList(), index)); } @@ -773,10 +765,17 @@ public void InsertRange(int index, IList items) /// public void Clear() { + if (_uncertainOrdinates.Count == 0) + return; + + bool wasSuppressed = SuppressCollectionChanged; + SuppressCollectionChanged = true; _uncertainOrdinates.Clear(); - if (SupressCollectionChanged == false) - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); + SuppressCollectionChanged = wasSuppressed; + _isValid = true; + if (SuppressCollectionChanged == false) + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); } /// diff --git a/Numerics/Data/Paired Data/UncertainOrdinate.cs b/Numerics/Data/Paired Data/UncertainOrdinate.cs index d3a81f7a..b605b13a 100644 --- a/Numerics/Data/Paired Data/UncertainOrdinate.cs +++ b/Numerics/Data/Paired Data/UncertainOrdinate.cs @@ -68,7 +68,9 @@ public UncertainOrdinate(double xValue, UnivariateDistributionBase yValue) { X = xValue; Y = yValue; - IsValid = !(double.IsInfinity(X) || double.IsNaN(X) || Y is null || !Y.ParametersValid); + IsValid = true; + if (double.IsInfinity(X) || double.IsNaN(X) || Y is null || Y.ParametersValid == false) + IsValid = false; } /// @@ -77,18 +79,18 @@ public UncertainOrdinate(double xValue, UnivariateDistributionBase yValue) /// The XElement to deserialize. public UncertainOrdinate(XElement xElement) { - var xAttr = xElement.Attribute(nameof(X)); double x = 0; UnivariateDistributionBase? dist = null; - if (xAttr != null) { double.TryParse(xAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out x); } - - var distEl = xElement.Element("Distribution"); - if (distEl != null) { dist = UnivariateDistributionFactory.CreateDistribution(distEl); } + var xAttr = xElement.Attribute(nameof(X)); + if (xAttr != null) double.TryParse(xAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out x); + var distElement = xElement.Element("Distribution"); + if (distElement != null) { dist = UnivariateDistributionFactory.CreateDistribution(distElement); } // X = x; Y = dist; - - IsValid = !(double.IsInfinity(X) || double.IsNaN(X) || Y is null || !Y.ParametersValid); + IsValid = true; + if (double.IsInfinity(X) || double.IsNaN(X) || Y is null || Y.ParametersValid == false) + IsValid = false; } /// @@ -99,8 +101,8 @@ public UncertainOrdinate(XElement xElement) public UncertainOrdinate(XElement xElement, UnivariateDistributionType distributionType) { double x = 0; - var xElAttr = xElement.Attribute(nameof(X)); - if (xElAttr != null) double.TryParse(xElAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out x); + var xAttr2 = xElement.Attribute(nameof(X)); + if (xAttr2 != null) double.TryParse(xAttr2.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out x); // backwards compatibility var dist = UnivariateDistributionFactory.CreateDistribution(distributionType); var props = dist.GetParameterPropertyNames; @@ -108,8 +110,8 @@ public UncertainOrdinate(XElement xElement, UnivariateDistributionType distribut for (int i = 0; i < props.Count(); i++) { double p = 0; - var xElPropsAttr = xElement.Attribute(props[i]); - if (xElPropsAttr != null) { double.TryParse(xElPropsAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out p); } + var paramAttr = xElement.Attribute(props[i]); + if (paramAttr != null) double.TryParse(paramAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out p); paramVals[i] = p; } dist.SetParameters(paramVals); @@ -153,10 +155,7 @@ public UncertainOrdinate(XElement xElement, UnivariateDistributionType distribut /// A 'sampled' ordinate value. public Ordinate GetOrdinate(double probability) { - if (Y is null) - { - throw new InvalidOperationException("Y distribution is not defined."); - } + if (Y is null) throw new InvalidOperationException("Cannot get ordinate when Y distribution is null."); return new Ordinate(X, Y.InverseCDF(probability)); } @@ -166,10 +165,7 @@ public Ordinate GetOrdinate(double probability) /// A mean ordinate value. public Ordinate GetOrdinate() { - if (Y is null) - { - throw new InvalidOperationException("Y distribution is not defined."); - } + if (Y is null) throw new InvalidOperationException("Cannot get ordinate when Y distribution is null."); return new Ordinate(X, Y.Mean); } @@ -186,17 +182,18 @@ public Ordinate GetOrdinate() /// A boolean indicating if the ordinate is valid or not given the criteria. public bool OrdinateValid(UncertainOrdinate ordinateToCompare, bool strictX, bool strictY, SortOrder xOrder, SortOrder yOrder, bool compareOrdinateIsNext, bool allowDifferentTypes = false) { - // + // if (IsValid == false) return false; if (ordinateToCompare.IsValid == false) return false; + // Both Y values must be non-null + if (Y is null || ordinateToCompare.Y is null) + return false; // Check for equivalent distribution types - if (allowDifferentTypes == false && (ordinateToCompare.Y is null || Y is null || ordinateToCompare.Y.Type != Y.Type)) + if (allowDifferentTypes == false && ordinateToCompare.Y.Type != Y.Type) return false; - if (Y is null || ordinateToCompare.Y is null) - return false; double minPercentile = Y.Type == UnivariateDistributionType.PertPercentile || Y.Type == UnivariateDistributionType.PertPercentileZ ? 0.05 : 1E-5; // Test reasonable lower bound @@ -246,16 +243,10 @@ public List OrdinateErrors(UncertainOrdinate ordinateToCompare, bool str } } // Check for equivalent distribution types - if(ordinateToCompare.Y is null || Y is null) - { - result.Add("Ordinate Y value must be defined."); - return result; - } - - if (allowDifferentTypes == false && ordinateToCompare.Y.Type != Y.Type) + if (Y is not null && ordinateToCompare.Y is not null && allowDifferentTypes == false && ordinateToCompare.Y.Type != Y.Type) result.Add("Can't compare two ordinates with different distribution types."); // Return False - // - if (IsValid == true && ordinateToCompare.IsValid == true) + // + if (IsValid == true && ordinateToCompare.IsValid == true && Y is not null && ordinateToCompare.Y is not null) { double minPercentile = Y.Type == UnivariateDistributionType.PertPercentile || Y.Type == UnivariateDistributionType.PertPercentileZ ? 0.05 : 1E-5; @@ -304,7 +295,7 @@ public List OrdinateErrors() /// True if two objects are numerically equal; otherwise, False. public static bool operator ==(UncertainOrdinate left, UncertainOrdinate right) { - + //if (left == null || right == null) return false; if (left.X != right.X) return false; if (left.Y is null && right.Y is null) @@ -351,7 +342,8 @@ public override int GetHashCode() { int hash = 17; hash = hash * 23 + X.GetHashCode(); - hash = hash * 23 + (Y is not null ? Y.GetHashCode() : 0); + if (Y is not null) + hash = hash * 23 + Y.GetHashCode(); return hash; } } @@ -364,9 +356,7 @@ public XElement ToXElement() var result = new XElement(nameof(UncertainOrdinate)); result.SetAttributeValue(nameof(X), X.ToString("G17", CultureInfo.InvariantCulture)); if (Y is not null) - { result.Add(Y.ToXElement()); - } return result; } diff --git a/Numerics/Data/Regression/LinearRegression.cs b/Numerics/Data/Regression/LinearRegression.cs index 50ec19cd..43615f8b 100644 --- a/Numerics/Data/Regression/LinearRegression.cs +++ b/Numerics/Data/Regression/LinearRegression.cs @@ -117,32 +117,32 @@ public LinearRegression(Matrix x, Vector y, bool hasIntercept = true) /// /// The list of estimated parameter values. /// - public List Parameters { get; private set; } = Array.Empty().ToList(); + public List Parameters { get; private set; } = null!; /// - /// The list of the estimated parameter names. + /// The list of the estimated parameter names. /// - public List ParameterNames { get; private set; } + public List ParameterNames { get; private set; } = null!; /// - /// The list of the estimated parameter standard errors. + /// The list of the estimated parameter standard errors. /// - public List ParameterStandardErrors { get; private set; } = Array.Empty().ToList(); + public List ParameterStandardErrors { get; private set; } = null!; /// /// The list of the estimated parameter t-statistics. /// - public List ParameterTStats { get; private set; } = Array.Empty().ToList(); + public List ParameterTStats { get; private set; } = null!; /// - /// The estimate parameter covariance matrix. + /// The estimate parameter covariance matrix. /// - public Matrix Covariance { get; private set; } = new Matrix(0, 0); + public Matrix Covariance { get; private set; } = null!; /// - /// The residuals of the fitted linear model. + /// The residuals of the fitted linear model. /// - public double[] Residuals { get; private set; } = Array.Empty(); + public double[] Residuals { get; private set; } = null!; /// /// The model standard error. @@ -273,6 +273,24 @@ private void FitSVD() } + /// + /// Prepares the design matrix for prediction by adding an intercept column if needed. + /// If the matrix already has the expected number of columns (e.g. internal X), it passes through. + /// If it has one fewer column and HasIntercept is true, the intercept column is added. + /// + /// The matrix of predictor values. + private Matrix PrepareDesignMatrix(Matrix x) + { + int expected = Parameters.Count; + if (x.NumberOfColumns == expected) + return x; + if (HasIntercept && x.NumberOfColumns == expected - 1) + return AddInterceptColumn(x); + throw new ArgumentException( + $"Expected {expected} columns{(HasIntercept ? $" (or {expected - 1} without intercept)" : "")}, but got {x.NumberOfColumns}.", + nameof(x)); + } + /// /// Helper method to add an intercept column to the covariate matrix. /// @@ -295,21 +313,10 @@ private static Matrix AddInterceptColumn(Matrix x) /// The matrix of predictor values. public double[] Predict(Matrix x) { - var result = new double[x.NumberOfRows]; - for (int i = 0; i < x.NumberOfRows; i++) - { - if (HasIntercept == true) - { - var values = new List() { 1 }; - values.AddRange(x.Row(i)); - result[i] = Tools.SumProduct(Parameters, values); - } - else - { - result[i] = Tools.SumProduct(Parameters, x.Row(i)); - } - } - + var xp = PrepareDesignMatrix(x); + var result = new double[xp.NumberOfRows]; + for (int i = 0; i < xp.NumberOfRows; i++) + result[i] = Tools.SumProduct(Parameters, xp.Row(i)); return result; } @@ -320,22 +327,12 @@ public double[] Predict(Matrix x) /// The confidence level; Default = 0.1, which will result in the 90% prediction intervals. public double[,] PredictionIntervals(Matrix x, double alpha = 0.1) { + var xp = PrepareDesignMatrix(x); var percentiles = new double[] { alpha / 2d, 1d - alpha / 2d }; - var result = new double[x.NumberOfRows, 3]; // lower, upper, mean - for (int i = 0; i < x.NumberOfRows; i++) + var result = new double[xp.NumberOfRows, 3]; // lower, upper, mean + for (int i = 0; i < xp.NumberOfRows; i++) { - double mu = 0; - if (HasIntercept == true) - { - var values = new List() { 1 }; - values.AddRange(x.Row(i)); - mu = Tools.SumProduct(Parameters, values); - - } - else - { - mu = Tools.SumProduct(Parameters, x.Row(i)); - } + double mu = Tools.SumProduct(Parameters, xp.Row(i)); var s = StandardError; var s2 = s * s; var n = DegreesOfFreedom; @@ -344,7 +341,6 @@ public double[] Predict(Matrix x) result[i, 1] = t.InverseCDF(percentiles[1]); result[i, 2] = mu; } - return result; } diff --git a/Numerics/Data/Statistics/Autocorrelation.cs b/Numerics/Data/Statistics/Autocorrelation.cs index c08daec1..e9c00b94 100644 --- a/Numerics/Data/Statistics/Autocorrelation.cs +++ b/Numerics/Data/Statistics/Autocorrelation.cs @@ -211,10 +211,9 @@ public enum Type if (lagMax < 0) lagMax = (int)Math.Floor(Math.Min(10d * Math.Log10(n), n - 1)); if (lagMax < 1 || n < 2) return null; var acf = Covariance(data, lagMax); - if (acf == null) return null; - double den = acf[0, 1]; + if (den == 0) return null; for (int i = 0; i < acf.GetLength(0); i++) acf[i, 1] /= den; return acf; @@ -234,11 +233,10 @@ public enum Type int n = timeSeries.Count; if (lagMax < 0) lagMax = (int)Math.Floor(Math.Min(10d * Math.Log10(n), n - 1)); if (lagMax < 1 || n < 2) return null; - var acf = Covariance(timeSeries, lagMax); if (acf == null) return null; - double den = acf[0, 1]; + if (den == 0) return null; for (int i = 0; i < acf.GetLength(0); i++) acf[i, 1] /= den; return acf; diff --git a/Numerics/Data/Statistics/BoxCox.cs b/Numerics/Data/Statistics/BoxCox.cs index 8d371516..e3f8d71b 100644 --- a/Numerics/Data/Statistics/BoxCox.cs +++ b/Numerics/Data/Statistics/BoxCox.cs @@ -133,6 +133,7 @@ public static double LogJacobian(IList values, double lambda) /// The transformation exponent. Range -5 to +5. public static double Transform(double value, double lambda) { + if (value <= 0) return double.NaN; if (Math.Abs(lambda) > 5d) return double.NaN; if (Math.Abs(lambda) < 1e-8) return Math.Log(value); return (Math.Pow(value, lambda) - 1.0d) / lambda; diff --git a/Numerics/Data/Statistics/GoodnessOfFit.cs b/Numerics/Data/Statistics/GoodnessOfFit.cs index 99b82642..ae5288b6 100644 --- a/Numerics/Data/Statistics/GoodnessOfFit.cs +++ b/Numerics/Data/Statistics/GoodnessOfFit.cs @@ -804,8 +804,8 @@ public static double PBIAS(IList observedValues, IList modeledVa /// Lower RSR values indicate better model performance. /// /// - /// Note: This method uses sample standard deviation (N-1 denominator) for the observed values, - /// which is consistent with the R hydroGOF package implementation. + /// Note: This method uses the population standard deviation (N denominator) for both RMSE and + /// observed variability, ensuring RSR = 1.0 when predictions equal the observed mean. /// /// public static double RSR(IList observedValues, IList modeledValues) @@ -828,7 +828,7 @@ public static double RSR(IList observedValues, IList modeledValu double rmse = Math.Sqrt(sse / n); double obsMean = sumObs / n; - // Second pass to compute sample standard deviation (using N-1) + // Second pass to compute standard deviation double variance = 0d; for (int i = 0; i < n; i++) { diff --git a/Numerics/Data/Statistics/Histogram.cs b/Numerics/Data/Statistics/Histogram.cs index 201ff926..806198e9 100644 --- a/Numerics/Data/Statistics/Histogram.cs +++ b/Numerics/Data/Statistics/Histogram.cs @@ -121,8 +121,7 @@ public double Midpoint /// public int CompareTo(Bin? other) { - if (other is null) { return 1; } - + if (other is null) return 1; if (UpperBound > other.LowerBound && LowerBound < other.LowerBound) { throw new ArgumentException(nameof(other), "The bins cannot be overlapping."); @@ -178,6 +177,26 @@ public override int GetHashCode() } } + /// + /// Internal constructor for JSON deserialization. Reconstructs a histogram from serialized bin data + /// without requiring the original raw data. + /// + /// The lower bound of the histogram. + /// The upper bound of the histogram. + /// The number of bins. + /// The bin width. + /// The list of histogram bins. + internal Histogram(double lowerBound, double upperBound, int numberOfBins, + double binWidth, List bins) + { + LowerBound = lowerBound; + UpperBound = upperBound; + NumberOfBins = numberOfBins; + BinWidth = binWidth; + foreach (var bin in bins) + AddBin(bin); + } + /// /// Constructs a histogram based on the data provided. The Rice Rule is used to set the bin sizes. /// @@ -189,6 +208,7 @@ public Histogram(IList data) // Get the bin boundaries LowerBound = data.Min(); UpperBound = data.Max(); + if (UpperBound == LowerBound) UpperBound = LowerBound + 1.0; BinWidth = (UpperBound - LowerBound) / NumberOfBins; // Add bins double xl = LowerBound; @@ -217,6 +237,7 @@ public Histogram(IList data, int numberOfBins) NumberOfBins = numberOfBins; LowerBound = data.Min(); UpperBound = data.Max(); + if (UpperBound == LowerBound) UpperBound = LowerBound + 1.0; BinWidth = (UpperBound - LowerBound) / NumberOfBins; // Add bins double xl = LowerBound; @@ -322,6 +343,7 @@ public double Median int total = 0; for (int i = 0; i < _bins.Count; i++) total += _bins[i].Frequency; + if (total == 0) return double.NaN; int halfTotal = (int)(total / 2d); int m = 0; int v = 0; @@ -458,7 +480,7 @@ public int GetBinIndexOf(double value) SortBins(); if (value < _bins.First().LowerBound || value > _bins.Last().UpperBound) { - throw new ArgumentException("value", "The value is not contained with the histogram limits."); + throw new ArgumentException("The value is not contained with the histogram limits.", nameof(value)); } int idx = Search.Bisection(value, _binLimits); return idx < 0 ? 0 : idx >= _binLimits.Count ? _binLimits.Count - 1 : idx; diff --git a/Numerics/Data/Statistics/HypothesisTests.cs b/Numerics/Data/Statistics/HypothesisTests.cs index 6c50254b..456d5311 100644 --- a/Numerics/Data/Statistics/HypothesisTests.cs +++ b/Numerics/Data/Statistics/HypothesisTests.cs @@ -28,13 +28,14 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -using System; -using System.Collections.Generic; -using System.Linq; using Numerics.Distributions; using Numerics.MachineLearning; using Numerics.Mathematics.LinearAlgebra; using Numerics.Mathematics.SpecialFunctions; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; namespace Numerics.Data.Statistics { @@ -65,6 +66,7 @@ public class HypothesisTests /// Returns the 2-sided p-value of the test statistic. public static double OneSampleTtest(IList sample, double populationMean = 0d) { + if (sample.Count < 2) throw new ArgumentException("Sample must have at least 2 observations.", nameof(sample)); var meanVar = Statistics.MeanVariance(sample); int N = sample.Count; double se = Math.Sqrt(meanVar.Item2) / Math.Sqrt(N); @@ -82,6 +84,7 @@ public static double OneSampleTtest(IList sample, double populationMean /// Returns the 2-sided p-value of the test statistic. public static double EqualVarianceTtest(IList sample1, IList sample2) { + if (sample1.Count + sample2.Count < 3) throw new ArgumentException("Combined sample size must be at least 3."); var meanVar1 = Statistics.MeanVariance(sample1); var meanVar2 = Statistics.MeanVariance(sample2); int N1 = sample1.Count; @@ -109,7 +112,7 @@ public static double UnequalVarianceTtest(IList sample1, IList s int n2 = sample2.Count; double t = Math.Abs((ave1 - ave2) / Math.Sqrt(var1 / n1 + var2 / n2)); double df = Tools.Sqr(var1 / n1 + var2 / n2) / (Tools.Sqr(var1 / n1) / (n1 - 1) + Tools.Sqr(var2 / n2) / (n2 - 1)); - var tdist = new StudentT((int)df); + var tdist = new StudentT(df); return (1d - tdist.CDF(t)) * 2d; } @@ -132,7 +135,7 @@ public static double PairedTtest(IList sample1, IList sample2) cov /= (df = n - 1); sd = Math.Sqrt((var1 + var2 - 2.0 * cov) / n); double t = Math.Abs((ave1 - ave2) / sd); - var tdist = new StudentT((int)df); + var tdist = new StudentT(df); return (1d - tdist.CDF(t)) * 2d; } @@ -144,10 +147,12 @@ public static double PairedTtest(IList sample1, IList sample2) /// Returns the p-value of the test statistic. public static double Ftest(IList sample1, IList sample2) { + if (sample1.Count < 2 || sample2.Count < 2) throw new ArgumentException("Each sample must have at least 2 observations."); var meanVar1 = Statistics.MeanVariance(sample1); var meanVar2 = Statistics.MeanVariance(sample2); int n1 = sample1.Count, n2 = sample2.Count; double ave1 = meanVar1.Item1, var1 = meanVar1.Item2, ave2 = meanVar2.Item1, var2 = meanVar2.Item2, df1, df2, f, pVal; + if (var1 == 0 && var2 == 0) return 1.0; if (var1 > var2) { f = var1 / var2; @@ -177,6 +182,8 @@ public static double Ftest(IList sample1, IList sample2) /// The p-value of the test statistic. public static void FtestModels(double sseRestricted, double sseFull, int dfRestricted, int dfFull, out double fStat, out double pValue) { + if (dfRestricted == dfFull) throw new ArgumentException("Restricted and full model degrees of freedom cannot be equal."); + if (dfFull <= 0) throw new ArgumentException("Full model degrees of freedom must be positive.", nameof(dfFull)); fStat = ((sseRestricted - sseFull) / (dfRestricted - dfFull)) / (sseFull / dfFull); pValue = 2.0 * Beta.Incomplete(0.5 * dfFull, 0.5 * dfRestricted, dfFull / (dfFull + dfRestricted * fStat)); if (pValue > 1.0) pValue = 2d - pValue; @@ -245,7 +252,7 @@ public static double LjungBoxTest(IList sample, int lagMax = -1) int n = sample.Count; if (lagMax < 0) lagMax = (int)Math.Floor(Math.Min(10d * Math.Log10(n), n - 1)); var acf = Autocorrelation.Function(sample, lagMax, Autocorrelation.Type.Correlation); - if (acf == null) throw new Exception("Autocorrelation function could not be calculated."); + if (acf == null) return double.NaN; double Q = 0; for (int k = 1; k <= lagMax; k++) Q += Tools.Sqr(acf[k, 1]) / (n - k); @@ -329,7 +336,6 @@ public static double LinearTrendTest(IList indices, IList sample var yVals = new Vector(sample.ToArray()); var lm = new LinearRegression(xVals, yVals, true); var tdist = new StudentT(lm.DegreesOfFreedom); - double d = Math.Abs(lm.Parameters[1] / lm.ParameterStandardErrors[1]); return (1 - tdist.CDF(Math.Abs(lm.Parameters[1] / lm.ParameterStandardErrors[1]))) * 2; } @@ -343,29 +349,36 @@ public static double UnimodalityTest(IList sample) int n = sample.Count; if (n < 10) throw new ArgumentException("The sample size must be greater than or equal to 10."); - // Fit 1-component GMM (unimodal) - var gmm1 = new GaussianMixtureModel(sample.ToArray(), 1); - gmm1.Train(12345, true); - var logLH1 = gmm1.LogLikelihood; + try + { + // Fit 1-component GMM (unimodal) + var gmm1 = new GaussianMixtureModel(sample.ToArray(), 1); + gmm1.Train(12345, true); + var logLH1 = gmm1.LogLikelihood; - // Fit 2-component GMM (potentially bimodal) - var gmm2 = new GaussianMixtureModel(sample.ToArray(), 2); - gmm2.Train(12345, true); - var logLH2 = gmm2.LogLikelihood; + // Fit 2-component GMM (potentially bimodal) + var gmm2 = new GaussianMixtureModel(sample.ToArray(), 2); + gmm2.Train(12345, true); + var logLH2 = gmm2.LogLikelihood; - // Compute test statistic: Likelihood Ratio Test - double testStat = 2 * (logLH2 - logLH1); - int df = 3; + // Compute test statistic: Likelihood Ratio Test + double testStat = 2 * (logLH2 - logLH1); + int df = 3; - // Compute p-value from chi-square distribution - var chiSquared = new ChiSquared(df); - double pval = 1.0 - chiSquared.CDF(testStat); + // Compute p-value from chi-square distribution + var chiSquared = new ChiSquared(df); + double pval = 1.0 - chiSquared.CDF(testStat); - return pval; + return pval; + } + catch (Exception ex) + { + Debug.WriteLine($"Error in hypothesis testing: {ex.Message}"); + return double.NaN; + } } - } diff --git a/Numerics/Data/Statistics/MultipleGrubbsBeckTest.cs b/Numerics/Data/Statistics/MultipleGrubbsBeckTest.cs index a714bd41..fb7f86c2 100644 --- a/Numerics/Data/Statistics/MultipleGrubbsBeckTest.cs +++ b/Numerics/Data/Statistics/MultipleGrubbsBeckTest.cs @@ -173,10 +173,6 @@ public static int Function(double[] X) return MGBTP; } - private static int _nIn; - private static int _rIn; - private static double _etaIn; - /// /// Auxiliary routine used to compute p-values (GGCRITP) for a Generalized Grubbs-Beck Test. /// @@ -187,16 +183,10 @@ private static double GGBCRITP(int N, int R, double ETA) { return 0.5d; } - else - { - _nIn = N; - _rIn = R; - _etaIn = ETA; - } - // The original FORTRAN source code utilized a globally adaptive Gauss-Kronrod integration method. + // The original FORTRAN source code utilized a globally adaptive Gauss-Kronrod integration method. // The number of low outliers computed by this method is consistent with the results from the FORTRAN code. - var sr = new AdaptiveGaussKronrod(FGGB, 1E-16, 1 - 1E-16); + var sr = new AdaptiveGaussKronrod((pzr) => FGGB(pzr, N, R, ETA), 1E-16, 1 - 1E-16); sr.MaxDepth = 25; sr.ReportFailure = false; sr.Integrate(); @@ -206,16 +196,13 @@ private static double GGBCRITP(int N, int R, double ETA) /// /// Auxiliary routine used in GGBCRITP /// - private static double FGGB(double PZR) + private static double FGGB(double PZR, int N, int R, double ETA) { double df, MuM, MuS2, VarM, VarS2, CovMS2; double EX1, EX2, EX3, EX4; double CovMS, VarS, alpha, beta; double MuMP, EtaP, H, Lambda, MuS, ncp, q, VarMP, PR, ZR, N2; double ANS; - int N = _nIn; - int R = _rIn; - double ETA = _etaIn; // Compute the value of the r-th smallest obs. based on its order statistic N2 = N - R; @@ -246,8 +233,19 @@ private static double FGGB(double PZR) df = 2.0d * alpha; ncp = (MuMP - ZR) / Math.Sqrt(VarMP); q = -Math.Sqrt(MuS2 / VarMP) * EtaP; - var NCTDist = new NoncentralT(df, ncp); - ANS = 1.0d - NCTDist.CDF(q); + // Match Fortran FP_TNC_CDF: use normal approximation for df > 20 + double cdfResult; + if (df > 20.0d) + { + double Z = (q * (1.0d - 1.0d / (4.0d * df)) - ncp) / Math.Sqrt(1.0d + q * q / (2.0d * df)); + cdfResult = Normal.StandardCDF(Z); + } + else + { + var NCTDist = new NoncentralT(df, ncp); + cdfResult = NCTDist.CDF(q); + } + ANS = 1.0d - cdfResult; return ANS; } @@ -261,6 +259,7 @@ public static void GrubbsBeckTest(IList sample, out double XHi, out doub { // The following polynomial approximation proposed by Pilon et al. (1985) int n = sample.Count; + if (sample.Any(x => x <= 0)) throw new ArgumentException("All sample values must be positive for the Grubbs-Beck test.", nameof(sample)); var logSample = new double[n]; for (int i = 0; i < n; i++) logSample[i] = Math.Log(sample[i]); double mean = Statistics.Mean(logSample); diff --git a/Numerics/Data/Statistics/Statistics.cs b/Numerics/Data/Statistics/Statistics.cs index 1ab3e9af..0a4e9056 100644 --- a/Numerics/Data/Statistics/Statistics.cs +++ b/Numerics/Data/Statistics/Statistics.cs @@ -210,7 +210,10 @@ public static double HarmonicMean(IList data) double sum = 0; for (int i = 0; i < data.Count; i++) + { + if (data[i] <= 0) return double.NaN; sum += 1.0 / data[i]; + } return data.Count / sum; } diff --git a/Numerics/Data/Statistics/YeoJohnson.cs b/Numerics/Data/Statistics/YeoJohnson.cs index ab6d060e..ec993b4f 100644 --- a/Numerics/Data/Statistics/YeoJohnson.cs +++ b/Numerics/Data/Statistics/YeoJohnson.cs @@ -156,10 +156,6 @@ public static double LogJacobian(IList values, double lambda) // Avoid log of zero or negative values logJacobianSum += Math.Log(Math.Abs(dTdy)); - //if (dTdy > 0) - // logJacobianSum += Math.Log(Math.Abs(dTdy)); - //else - // return double.NegativeInfinity; // log-likelihood undefined } return logJacobianSum; } diff --git a/Numerics/Data/Time Series/Support/Series.cs b/Numerics/Data/Time Series/Support/Series.cs index a78cd46f..58627240 100644 --- a/Numerics/Data/Time Series/Support/Series.cs +++ b/Numerics/Data/Time Series/Support/Series.cs @@ -69,7 +69,7 @@ public SeriesOrdinate this[int index] var oldvalue = _seriesOrdinates[index]; _seriesOrdinates[index] = value; if (SuppressCollectionChanged == false) - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, value, oldvalue)); + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, value, oldvalue, index)); } } } @@ -80,8 +80,7 @@ public SeriesOrdinate this[int index] get { return _seriesOrdinates[index]; } set { - if (value is null) { throw new ArgumentNullException(nameof(value)); } - + if (value is null) throw new ArgumentNullException(nameof(value)); if (value.GetType() != typeof(SeriesOrdinate)) { if (_seriesOrdinates[index] != (SeriesOrdinate)value) @@ -89,10 +88,9 @@ public SeriesOrdinate this[int index] var oldvalue = _seriesOrdinates[index]; _seriesOrdinates[index] = (SeriesOrdinate)value; if (SuppressCollectionChanged == false) - CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, value, oldvalue)); + CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Replace, value, oldvalue, index)); } } - } } @@ -110,8 +108,10 @@ public SeriesOrdinate this[int index] /// public bool IsFixedSize => false; + private readonly object _syncRoot = new object(); + /// - public object SyncRoot => _seriesOrdinates.Count > 0 ? _seriesOrdinates[0]! : new object(); + public object SyncRoot => _syncRoot; /// public bool IsSynchronized => false; @@ -166,7 +166,7 @@ public virtual bool Remove(SeriesOrdinate item) /// public void Remove(object? item) { - if (item is null) throw new ArgumentNullException(nameof(item)); + if (item is null) return; if (item.GetType() == typeof(SeriesOrdinate)) { Remove((SeriesOrdinate)item); @@ -202,7 +202,7 @@ public bool Contains(SeriesOrdinate item) /// public bool Contains(object? item) { - if (item is null) throw new ArgumentNullException(nameof(item)); + if (item is null) return false; if (item.GetType() == typeof(SeriesOrdinate)) { return Contains((SeriesOrdinate)item); @@ -234,7 +234,7 @@ public int IndexOf(SeriesOrdinate item) /// public int IndexOf(object? item) { - if (item is null) throw new ArgumentNullException(nameof(item)); + if (item is null) return -1; if (item.GetType() == typeof(SeriesOrdinate)) { return _seriesOrdinates.IndexOf((SeriesOrdinate)item); diff --git a/Numerics/Data/Time Series/Support/SeriesOrdinate.cs b/Numerics/Data/Time Series/Support/SeriesOrdinate.cs index 8cbde3f1..a1471e76 100644 --- a/Numerics/Data/Time Series/Support/SeriesOrdinate.cs +++ b/Numerics/Data/Time Series/Support/SeriesOrdinate.cs @@ -50,11 +50,7 @@ public class SeriesOrdinate : INotifyPropertyChanged, IEquatable /// /// Constructs a new series ordinate. /// - public SeriesOrdinate() - { - _index = default!; - _value = default!; - } + public SeriesOrdinate() { } /// /// Constructs a new series ordinate. @@ -70,12 +66,12 @@ public SeriesOrdinate(TIndex index, TValue value) /// /// Protected index property. /// - protected TIndex _index; + protected TIndex _index = default!; /// /// Protected value property. /// - protected TValue _value; + protected TValue _value = default!; /// public event PropertyChangedEventHandler? PropertyChanged; @@ -122,7 +118,7 @@ public bool Equals(SeriesOrdinate? other) } /// - public override bool Equals(object? obj) => Equals((SeriesOrdinate?)obj); + public override bool Equals(object? obj) => Equals(obj as SeriesOrdinate); /// /// Equality operator overload. @@ -147,8 +143,8 @@ public override int GetHashCode() unchecked { int hash = 17; - hash = hash * 23 + (_index is null ? 0 : EqualityComparer.Default.GetHashCode(_index)); - hash = hash * 23 + (_value is null ? 0 : EqualityComparer.Default.GetHashCode(_value)); + hash = hash * 23 + EqualityComparer.Default.GetHashCode(_index!); + hash = hash * 23 + EqualityComparer.Default.GetHashCode(_value!); return hash; } } diff --git a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs index dff08c99..40be7c6e 100644 --- a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs +++ b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs @@ -34,6 +34,8 @@ using System.Net; using System.Xml.Linq; using System; +using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using System.Net.Http; using System.Globalization; @@ -53,6 +55,14 @@ namespace Numerics.Data /// public class TimeSeriesDownload { + private static readonly HttpClient _defaultClient = new HttpClient() { Timeout = TimeSpan.FromSeconds(60) }; + private static readonly HttpClient _decompressClient = new HttpClient(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }) + { Timeout = TimeSpan.FromSeconds(60) }; + + private const string UserAgent = "USACE-Numerics/2.0"; /// /// Checks if there is an Internet connection. @@ -61,10 +71,9 @@ public static async Task IsConnectedToInternet() { try { - using (HttpClient client = new HttpClient()) + using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) { - client.Timeout = TimeSpan.FromSeconds(5); - await client.GetAsync("https://www.google.com"); + await _defaultClient.GetAsync("https://waterservices.usgs.gov/nwis/", cts.Token); return true; } } @@ -110,7 +119,15 @@ public enum TimeSeriesType /// /// Annual max peak stage. /// - PeakStage + PeakStage, + /// + /// Discrete field measurement of discharge, used for rating curves. + /// + MeasuredDischarge, + /// + /// Discrete field measurement of stage (gage height), used for rating curves. + /// + MeasuredStage } /// @@ -176,9 +193,9 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType // Check site number - if (siteNumber.Length != 11) + if (siteNumber.Length != 11 || !Regex.IsMatch(siteNumber, @"^[A-Za-z0-9]{11}$")) { - throw new ArgumentException("The GHCN site number must be 11-digits long", nameof(siteNumber)); + throw new ArgumentException("The GHCN site number must be exactly 11 alphanumeric characters.", nameof(siteNumber)); } // Check time series type @@ -189,7 +206,7 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType var timeSeries = new TimeSeries(TimeInterval.OneDay); DateTime? previousDate = null; - string tempFilePath = Path.Combine(Path.GetTempPath(), $"{siteNumber}.dly"); + string tempFilePath = Path.Combine(Path.GetTempPath(), $"{Path.GetRandomFileName()}.dly"); // Check internet connection if (!await IsConnectedToInternet()) @@ -202,9 +219,9 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType // Download the GHCN file string ghcnBaseUrl = "https://www.ncei.noaa.gov/pub/data/ghcn/daily/all/"; - string stationFileUrl = $"{ghcnBaseUrl}{siteNumber}.dly"; - using (var client = new HttpClient()) + string stationFileUrl = $"{ghcnBaseUrl}{Uri.EscapeDataString(siteNumber)}.dly"; { + var client = _defaultClient; // Request the file and ensure that response headers are read first. using (HttpResponseMessage response = await client.GetAsync(stationFileUrl, HttpCompletionOption.ResponseHeadersRead)) { @@ -268,10 +285,6 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType } } - catch (Exception) - { - throw; - } finally { // Ensure temporary file is deleted even if an exception occurs @@ -335,7 +348,14 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType return $"https://nwis.waterdata.usgs.gov/nwis/peak?site_no={siteNumber}&agency_cd=USGS&format=rdb"; } - // Determine URL parts for daily and instantaneous data + // For field measurements (modernized OGC API) + if (timeSeriesType == TimeSeriesType.MeasuredDischarge || timeSeriesType == TimeSeriesType.MeasuredStage) + { + string paramCode = timeSeriesType == TimeSeriesType.MeasuredDischarge ? "00060" : "00065"; + return $"https://api.waterdata.usgs.gov/ogcapi/v0/collections/field-measurements/items?monitoring_location_id=USGS-{siteNumber}¶meter_code={paramCode}&limit=10000&f=json"; + } + + // Daily and instantaneous data (legacy NWIS API) string dataTypePart = (timeSeriesType == TimeSeriesType.DailyDischarge || timeSeriesType == TimeSeriesType.DailyStage) ? "dv/?" : "iv/?"; string siteNumberPart = $"&sites={siteNumber}"; string parameterCodePart = $"¶meterCd={(timeSeriesType == TimeSeriesType.DailyDischarge || timeSeriesType == TimeSeriesType.InstantaneousDischarge ? "00060" : "00065")}"; @@ -375,37 +395,35 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType throw new InvalidOperationException("No internet connection."); } - var timeSeries = new TimeSeries(); + var timeSeries = (timeSeriesType == TimeSeriesType.MeasuredDischarge || timeSeriesType == TimeSeriesType.MeasuredStage || + timeSeriesType == TimeSeriesType.InstantaneousDischarge || timeSeriesType == TimeSeriesType.InstantaneousStage) + ? new TimeSeries(TimeInterval.Irregular) + : new TimeSeries(); string textDownload = ""; // Get URL string url = CreateURLForUSGSDownload(siteNumber, timeSeriesType); - try + // For daily or instantaneous data, use HttpClient with gzip support { - // For daily or instantaneous data, use HttpClient with gzip support - if (timeSeriesType == TimeSeriesType.DailyDischarge || - timeSeriesType == TimeSeriesType.DailyStage || - timeSeriesType == TimeSeriesType.InstantaneousDischarge || + if (timeSeriesType == TimeSeriesType.DailyDischarge || + timeSeriesType == TimeSeriesType.DailyStage || + timeSeriesType == TimeSeriesType.InstantaneousDischarge || timeSeriesType == TimeSeriesType.InstantaneousStage) { - using (HttpClient client = new HttpClient()) + bool isInstantaneous = timeSeriesType == TimeSeriesType.InstantaneousDischarge || + timeSeriesType == TimeSeriesType.InstantaneousStage; + // Instantaneous RDB has an extra tz_cd column at index 3, pushing the value to index 4 + int valueIndex = isInstantaneous ? 4 : 3; + { - // Set request headers to accept gzip encoding - client.DefaultRequestHeaders.AcceptEncoding.Add(new System.Net.Http.Headers.StringWithQualityHeaderValue("gzip")); + var client = _decompressClient; HttpResponseMessage response = await client.GetAsync(url); response.EnsureSuccessStatusCode(); - // Ensure the response content is compressed as expected - if (!response.Content.Headers.ContentEncoding.Contains("gzip")) - { - throw new Exception("Response is not compressed as expected."); - } - - using (Stream compressedStream = await response.Content.ReadAsStreamAsync()) - using (GZipStream decompressionStream = new GZipStream(compressedStream, CompressionMode.Decompress)) - using (StreamReader reader = new StreamReader(decompressionStream)) + using (Stream contentStream = await response.Content.ReadAsStreamAsync()) + using (StreamReader reader = new StreamReader(contentStream)) { string? line; bool isHeader = true; @@ -431,18 +449,17 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType string[] fields = line.Split('\t'); // Validate expected number of fields and record type - if (fields.Length < 5 || fields[0] != "USGS") + if (fields.Length <= valueIndex || fields[0] != "USGS") continue; // Parse date (assume fields[2] contains the date) if (!DateTime.TryParse(fields[2], out DateTime index)) { - // Optionally log or handle date parse failures continue; } - // Fill in missing days if the time series is not continuous - if (timeSeries.Count > 0 && index != TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay)) + // Fill in missing days only for daily series (not instantaneous) + if (!isInstantaneous && timeSeries.Count > 0 && index != TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay)) { while (timeSeries.Last().Index < TimeSeries.SubtractTimeInterval(index, TimeInterval.OneDay)) { @@ -452,7 +469,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType } // Get and parse value - string valueStr = fields[3]; + string valueStr = fields[valueIndex]; double value = string.IsNullOrWhiteSpace(valueStr) ? double.NaN : double.TryParse(valueStr, out double tempVal) ? tempVal : double.NaN; timeSeries.Add(new SeriesOrdinate(index, value)); } @@ -462,10 +479,9 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType // For peak data (annual max values) else if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) { - using (HttpClient client = new HttpClient()) { // Download data as string (assumes USGS peak data is not compressed) - textDownload = await client.GetStringAsync(url); + textDownload = await _defaultClient.GetStringAsync(url); var lines = textDownload.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries); foreach (string line in lines) @@ -514,38 +530,125 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType } } } - } - catch (Exception) - { - throw; + // For field measurements (modernized OGC API) + else if (timeSeriesType == TimeSeriesType.MeasuredDischarge || timeSeriesType == TimeSeriesType.MeasuredStage) + { + { + var rawText = new System.Text.StringBuilder(); + await ParseUSGSOgcApiPages(_decompressClient, url, timeSeries, rawText); + textDownload = rawText.ToString(); + } + + timeSeries.SortByTime(); + } } return (timeSeries, textDownload); } + /// + /// Parse paginated USGS OGC API responses (GeoJSON FeatureCollection). + /// Follows cursor-based pagination via links[rel=next]. + /// + /// HttpClient to use for requests. + /// The initial API URL. + /// TimeSeries to populate with parsed data. + /// Optional StringBuilder to accumulate raw JSON responses. + private static async Task ParseUSGSOgcApiPages(HttpClient client, string initialUrl, TimeSeries timeSeries, System.Text.StringBuilder? rawText = null) + { + string? nextUrl = initialUrl; + + while (nextUrl != null) + { + // Retry with exponential backoff for rate limiting (429) + HttpResponseMessage response = await client.GetAsync(nextUrl); + for (int attempt = 1; attempt < 6; attempt++) + { + if ((int)response.StatusCode != 429) break; + int delayMs = Math.Min((int)Math.Pow(2, attempt) * 2000, 60000); + await System.Threading.Tasks.Task.Delay(delayMs); + response = await client.GetAsync(nextUrl); + } + response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadAsStringAsync(); + + if (rawText != null) + { + if (rawText.Length > 0) rawText.AppendLine(); + rawText.Append(json); + } + + var doc = System.Text.Json.JsonDocument.Parse(json); + var root = doc.RootElement; + + if (root.TryGetProperty("features", out var features)) + { + foreach (var feature in features.EnumerateArray()) + { + var props = feature.GetProperty("properties"); + string? timeStr = props.GetProperty("time").GetString(); + if (!DateTime.TryParse(timeStr, CultureInfo.InvariantCulture, + DateTimeStyles.AdjustToUniversal, out DateTime dt)) + continue; + + string? valueStr = props.GetProperty("value").GetString(); + if (string.IsNullOrWhiteSpace(valueStr) || + !double.TryParse(valueStr, NumberStyles.Float, + CultureInfo.InvariantCulture, out double val)) + continue; + + timeSeries.Add(new SeriesOrdinate(dt, val)); + } + } + + // Follow cursor-based pagination + nextUrl = null; + if (root.TryGetProperty("links", out var links)) + { + foreach (var link in links.EnumerateArray()) + { + if (link.GetProperty("rel").GetString() is "next") + { + var candidateUrl = link.GetProperty("href").GetString(); + // Validate the URL domain to prevent open redirect + if (candidateUrl != null && Uri.TryCreate(candidateUrl, UriKind.Absolute, out var uri) && + uri.Host.EndsWith(".usgs.gov", StringComparison.OrdinalIgnoreCase)) + { + nextUrl = candidateUrl; + } + // Small delay between pages to avoid rate limiting + await System.Threading.Tasks.Task.Delay(200); + break; + } + } + } + } + } + #endregion #region Canadian Hydrometric Monitoring Network (CHMN) /// - /// Download historical daily means from the Water Survey of Canada + /// Download time series data from the Water Survey of Canada /// (Environment and Climate Change Canada Hydrometric Monitoring Network). /// /// WSC 7-character station id, e.g., "08LG010". /// - /// DailyDischarge → parameters[]=flow - /// DailyStage → parameters[]=level + /// DailyDischarge / DailyStage → daily_data endpoint (parameters[]=flow/level). + /// InstantaneousDischarge / InstantaneousStage → real_time_data endpoint (parameters[]=46/47). + /// PeakDischarge / PeakStage → peak_data endpoint (parameters[]=flow/level). /// - /// Desired discharge unit if DailyDischarge. - /// Desired stage unit if DailyStage. + /// Desired discharge unit for discharge types. + /// Desired stage unit for stage types. /// - /// Optional inclusive start date. If null, defaults to 1800-01-01 - /// to request full POR. The service returns only available days. + /// Optional inclusive start date. If null, defaults vary by type: + /// Daily → 1800-01-01, Instantaneous → 18 months ago, Peak → year 1800. /// /// - /// Optional inclusive end date. If null, defaults to DateTime.Today. + /// Optional inclusive end date. If null, defaults to DateTime.Today (or Now for instantaneous). /// - /// TimeSeries of daily means. + /// TimeSeries of values. public static async Task FromCHMN( string stationNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyDischarge, @@ -562,187 +665,273 @@ public static async Task FromCHMN( if (string.IsNullOrWhiteSpace(stationNumber) || stationNumber.Length != 7) throw new ArgumentException("The WSC station number must be 7 characters, e.g., 08LG010.", nameof(stationNumber)); - // Supported series types for this endpoint - if (timeSeriesType != TimeSeriesType.DailyDischarge && timeSeriesType != TimeSeriesType.DailyStage) - throw new ArgumentException("Canadian daily_data API supports DailyDischarge or DailyStage.", nameof(timeSeriesType)); - - // Parameter mapping per WSC docs: parameters[]=flow or parameters[]=level - string parameter = timeSeriesType == TimeSeriesType.DailyDischarge ? "flow" : "level"; - - // Date range: request wide window to effectively get full POR - DateTime sd = startDate ?? new DateTime(1800, 1, 1); - DateTime ed = endDate ?? DateTime.Today; + // Supported series types + if (timeSeriesType == TimeSeriesType.DailyPrecipitation || timeSeriesType == TimeSeriesType.DailySnow || + timeSeriesType == TimeSeriesType.MeasuredDischarge || timeSeriesType == TimeSeriesType.MeasuredStage) + throw new ArgumentException("Canadian API supports Daily, Instantaneous, and Peak discharge/stage types. Field measurements are not available through the API.", nameof(timeSeriesType)); - string url = - "https://wateroffice.ec.gc.ca/services/daily_data/csv/inline" + - $"?stations[]={Uri.EscapeDataString(stationNumber)}" + - $"¶meters[]={parameter}" + - $"&start_date={sd:yyyy-MM-dd}" + - $"&end_date={ed:yyyy-MM-dd}"; + // Determine if this is a discharge or stage request + bool isDischarge = timeSeriesType == TimeSeriesType.DailyDischarge || + timeSeriesType == TimeSeriesType.InstantaneousDischarge || + timeSeriesType == TimeSeriesType.PeakDischarge; - var ts = new TimeSeries(TimeInterval.OneDay); + // Build the URL based on the time series type + string url; + TimeInterval interval; - // Create HttpClientHandler to bypass proxy if needed - var handler = new HttpClientHandler + if (timeSeriesType == TimeSeriesType.DailyDischarge || timeSeriesType == TimeSeriesType.DailyStage) { - UseProxy = false, // Bypass corporate proxy - AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate - }; + // Daily data endpoint + string parameter = isDischarge ? "flow" : "level"; + DateTime sd = startDate ?? new DateTime(1800, 1, 1); + DateTime ed = endDate ?? DateTime.Today; + url = "https://wateroffice.ec.gc.ca/services/daily_data/csv/inline" + + $"?stations[]={Uri.EscapeDataString(stationNumber)}" + + $"¶meters[]={parameter}" + + $"&start_date={sd:yyyy-MM-dd}" + + $"&end_date={ed:yyyy-MM-dd}"; + interval = TimeInterval.OneDay; + } + else if (timeSeriesType == TimeSeriesType.InstantaneousDischarge || timeSeriesType == TimeSeriesType.InstantaneousStage) + { + // Real-time data endpoint (parameter codes: 46=level, 47=discharge) + string paramCode = isDischarge ? "47" : "46"; + DateTime sd = startDate ?? DateTime.UtcNow.AddMonths(-18); + DateTime ed = endDate ?? DateTime.UtcNow; + url = "https://wateroffice.ec.gc.ca/services/real_time_data/csv/inline" + + $"?stations[]={Uri.EscapeDataString(stationNumber)}" + + $"¶meters[]={paramCode}" + + $"&start_date={sd:yyyy-MM-dd HH:mm:ss}" + + $"&end_date={ed:yyyy-MM-dd HH:mm:ss}"; + interval = TimeInterval.FiveMinute; + } + else // PeakDischarge or PeakStage + { + // Peak data endpoint + string parameter = isDischarge ? "flow" : "level"; + int startYear = startDate?.Year ?? 1800; + int endYear = endDate?.Year ?? DateTime.Today.Year; + url = "https://wateroffice.ec.gc.ca/services/peak_data/csv/inline" + + $"?stations[]={Uri.EscapeDataString(stationNumber)}" + + $"¶meters[]={parameter}" + + $"&start_year={startYear}" + + $"&end_year={endYear}"; + interval = TimeInterval.Irregular; + } + + var ts = new TimeSeries(interval); - using (var client = new HttpClient(handler)) { - // Add browser-like headers to avoid security proxy issues - client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); - client.DefaultRequestHeaders.Add("Accept", "text/csv,application/csv,text/plain,*/*"); - client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); - client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br"); - client.DefaultRequestHeaders.Add("DNT", "1"); - client.DefaultRequestHeaders.Add("Connection", "keep-alive"); - client.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1"); + var client = _decompressClient; - // The endpoint returns CSV (possibly gzip-compressed) + // The endpoint returns CSV (automatically decompressed by HttpClientHandler) using HttpResponseMessage resp = await client.GetAsync(url); resp.EnsureSuccessStatusCode(); string csv; - byte[] responseBytes = await resp.Content.ReadAsByteArrayAsync(); - - // Check if the response is gzip-compressed by checking the magic number - // Gzip magic number is 0x1f 0x8b - bool isGzipped = responseBytes.Length >= 2 && responseBytes[0] == 0x1f && responseBytes[1] == 0x8b; + { + csv = await resp.Content.ReadAsStringAsync(); + } - if (isGzipped) + // Parse the CSV response based on endpoint type + if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) { - // Decompress gzip data - using (var compressedStream = new MemoryStream(responseBytes)) - using (var gzipStream = new GZipStream(compressedStream, CompressionMode.Decompress)) - using (var reader = new StreamReader(gzipStream)) - { - csv = await reader.ReadToEndAsync(); - } + // Peak data CSV format: + // ID,Parameter/Paramètre,Date,Timezone/Fuseau horaire,Type/Catégorie,Value/Valeur,Symbol/Symbole + ParseCHMNPeakCsv(csv, stationNumber, isDischarge, dischargeUnit, heightUnit, ts); } else { - // Read as plain text - csv = System.Text.Encoding.UTF8.GetString(responseBytes); + // Daily and real-time CSV share the same column layout: + // ID,Date,Parameter/Paramètre,Value/Valeur,... + ParseCHMNDailyCsv(csv, stationNumber, timeSeriesType, isDischarge, dischargeUnit, heightUnit, ts); } + } - // Parse the CSV response - using (var sr = new StringReader(csv)) - { - string? line; - int idCol = 0, dateCol = 1, paramCol = 2, valueCol = 3; - bool headerParsed = false; + return ts; + } - // For filling missing dates - DateTime? prevDate = null; + /// + /// Parse CHMN daily or real-time CSV data. + /// + /// + /// CSV format: ID,Date,Parameter/Paramètre,Value/Valeur,... (additional columns vary by endpoint) + /// + private static void ParseCHMNDailyCsv(string csv, string stationNumber, TimeSeriesType timeSeriesType, + bool isDischarge, DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts) + { + bool isDailyType = timeSeriesType == TimeSeriesType.DailyDischarge || timeSeriesType == TimeSeriesType.DailyStage; - while ((line = sr.ReadLine()) != null) - { - // Skip empty lines - if (string.IsNullOrWhiteSpace(line)) continue; + using (var sr = new StringReader(csv)) + { + string? line; + int idCol = 0, dateCol = 1, paramCol = 2, valueCol = 3; + bool headerParsed = false; + DateTime? prevDate = null; - // Try to detect and parse the header - if (!headerParsed) + while ((line = sr.ReadLine()) != null) + { + if (string.IsNullOrWhiteSpace(line)) continue; + + // Detect header line + if (!headerParsed) + { + if (line.Contains("ID") && line.Contains("Date") && line.Contains("Value")) { + headerParsed = true; + } + continue; + } - // If this line contains the key header terms, treat it as the header - if (line == " ID,Date,Parameter/Paramètre,Value/Valeur,Symbol/Symbole") - { - headerParsed = true; - } + // Parse data rows + string[] parts = line.Split(','); + if (parts.Length <= Math.Max(Math.Max(idCol, dateCol), valueCol)) + continue; - // If we haven't found a header yet, skip this line (could be metadata) - continue; - } + // Check station match + if (!parts[idCol].Trim().Equals(stationNumber, StringComparison.OrdinalIgnoreCase)) + continue; - // We're past the header - parse data rows - string[] parts = line.Split(','); + // Check parameter match for daily data (parameter column contains text like "discharge/débit" or "level") + if (isDailyType && paramCol >= 0 && parts.Length > paramCol) + { + string paramValue = parts[paramCol].Trim().ToLowerInvariant(); + bool isFlowParam = paramValue.Contains("discharge") || paramValue.Contains("débit") || paramValue.Contains("flow"); + bool isLevelParam = paramValue.Contains("level") || paramValue.Contains("stage") || paramValue.Contains("niveau"); - // Validate row has enough columns - if (parts.Length <= Math.Max(Math.Max(idCol, dateCol), valueCol)) - continue; + if (isDischarge && !isFlowParam) continue; + if (!isDischarge && !isLevelParam) continue; + } - // Check station match - if (!parts[idCol].Trim().Equals(stationNumber, StringComparison.OrdinalIgnoreCase)) - continue; + // Parse date + if (!DateTime.TryParse(parts[dateCol].Trim(), CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime date)) + continue; - // Check parameter match (parameter column contains "discharge/débit" or "level") - if (paramCol >= 0 && parts.Length > paramCol) + // Parse value + double val = double.NaN; + string valueStr = parts[valueCol].Trim(); + + if (!string.IsNullOrWhiteSpace(valueStr)) + { + if (double.TryParse(valueStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double raw)) { - string paramValue = parts[paramCol].Trim().ToLowerInvariant(); - bool isFlowParam = paramValue.Contains("discharge") || paramValue.Contains("débit") || paramValue.Contains("flow"); - bool isLevelParam = paramValue.Contains("level") || paramValue.Contains("stage") || paramValue.Contains("niveau"); - - // Skip if parameter doesn't match requested type - if (timeSeriesType == TimeSeriesType.DailyDischarge && !isFlowParam) - continue; - if (timeSeriesType == TimeSeriesType.DailyStage && !isLevelParam) - continue; + val = ConvertCHMNValue(raw, isDischarge, dischargeUnit, heightUnit); } + } - // Parse date (handle both M/d/yyyy and yyyy-MM-dd formats) - if (!DateTime.TryParse(parts[dateCol].Trim(), out DateTime date)) - continue; + // Fill gaps with NaN for daily series only + if (isDailyType && prevDate.HasValue && (date - prevDate.Value).Days > 1) + FillMissingDates(ts, prevDate.Value, date); + + ts.Add(new SeriesOrdinate(date, val)); + prevDate = date; + } + } + } + + /// + /// Parse CHMN peak data CSV. + /// + /// + /// CSV format: ID,Parameter/Paramètre,Date,Timezone/Fuseau horaire,Type/Catégorie,Value/Valeur,Symbol/Symbole. + /// Only "maximum" rows are included (minimums are skipped). + /// + private static void ParseCHMNPeakCsv(string csv, string stationNumber, bool isDischarge, + DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts) + { + using (var sr = new StringReader(csv)) + { + string? line; + bool headerParsed = false; + + // Peak CSV columns: ID(0), Parameter(1), Date(2), Timezone(3), Type(4), Value(5), Symbol(6) + int idCol = 0, dateCol = 2, typeCol = 4, valueCol = 5; - // Parse value - double val = double.NaN; - string valueStr = parts[valueCol].Trim(); + while ((line = sr.ReadLine()) != null) + { + if (string.IsNullOrWhiteSpace(line)) continue; - if (!string.IsNullOrWhiteSpace(valueStr)) + // Detect header line + if (!headerParsed) + { + if (line.Contains("ID") && line.Contains("Date") && line.Contains("Value")) { - // WSC uses dot decimal, values in m^3/s for flow and m for level - if (double.TryParse(valueStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double raw)) - { - if (timeSeriesType == TimeSeriesType.DailyDischarge) - { - val = dischargeUnit == DischargeUnit.CubicFeetPerSecond - ? raw * 35.3146667 // cms → cfs - : raw; // cms - } - else - { - val = heightUnit == HeightUnit.Feet - ? raw * 3.280839895 // m → ft - : raw; // m - } - } + headerParsed = true; } + continue; + } - // Fill gaps with NaN to keep a continuous daily series - if (prevDate.HasValue && (date - prevDate.Value).Days > 1) - FillMissingDates(ts, prevDate.Value, date); + string[] parts = line.Split(','); + if (parts.Length <= valueCol) continue; - ts.Add(new SeriesOrdinate(date, val)); - prevDate = date; - } + // Check station match + if (!parts[idCol].Trim().Equals(stationNumber, StringComparison.OrdinalIgnoreCase)) + continue; + + // Only include maximum values (skip minimums) + string typeValue = parts[typeCol].Trim().ToLowerInvariant(); + if (typeValue != "maximum") continue; + + // Parse date + if (!DateTime.TryParse(parts[dateCol].Trim(), CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime date)) + continue; + + // Parse value + string valueStr = parts[valueCol].Trim(); + if (string.IsNullOrWhiteSpace(valueStr)) continue; + if (!double.TryParse(valueStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double raw)) + continue; + + double val = ConvertCHMNValue(raw, isDischarge, dischargeUnit, heightUnit); + ts.Add(new SeriesOrdinate(date, val)); } } - - return ts; } + /// + /// Convert a CHMN raw value to the desired unit. + /// + /// + /// WSC native units: discharge in m³/s, stage in meters. + /// + private static double ConvertCHMNValue(double raw, bool isDischarge, DischargeUnit dischargeUnit, HeightUnit heightUnit) + { + if (isDischarge) + { + return dischargeUnit == DischargeUnit.CubicFeetPerSecond + ? raw * 35.3146667 // cms → cfs + : raw; // cms + } + else + { + return heightUnit == HeightUnit.Feet + ? raw * 3.280839895 // m → ft + : raw; // m + } + } #endregion #region Australian Bureau of Meteorology (BOM) /// - /// Download daily discharge or stage data from the Australian Bureau of Meteorology (BOM) + /// Download time series data from the Australian Bureau of Meteorology (BOM) /// via the KiWIS API. /// /// BOM station number (e.g., "410730"). - /// The time series type (DailyDischarge or DailyStage). - /// Desired discharge unit if DailyDischarge. - /// Desired stage unit if DailyStage. + /// The time series type. + /// Desired discharge unit for discharge types. + /// Desired stage unit for stage types. + /// Desired depth unit for precipitation types. /// Optional start date. If null, attempts to retrieve full period of record. /// Optional end date. If null, defaults to today. - /// TimeSeries of daily values. + /// TimeSeries of values. public static async Task FromABOM( string stationNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyDischarge, DischargeUnit dischargeUnit = DischargeUnit.CubicMetersPerSecond, HeightUnit heightUnit = HeightUnit.Meters, + DepthUnit depthUnit = DepthUnit.Millimeters, DateTime? startDate = null, DateTime? endDate = null) { @@ -752,9 +941,16 @@ public static async Task FromABOM( if (string.IsNullOrWhiteSpace(stationNumber) || stationNumber.Length < 6) throw new ArgumentException("BOM station number must be at least 6 digits.", nameof(stationNumber)); - // Validate time series type - if (timeSeriesType != TimeSeriesType.DailyDischarge && timeSeriesType != TimeSeriesType.DailyStage) - throw new ArgumentException("BOM API supports DailyDischarge or DailyStage only.", nameof(timeSeriesType)); + // Validate time series type - BOM supports daily/instantaneous discharge and stage, plus daily precipitation + var supportedTypes = new[] { + TimeSeriesType.DailyDischarge, TimeSeriesType.DailyStage, + TimeSeriesType.InstantaneousDischarge, TimeSeriesType.InstantaneousStage, + TimeSeriesType.DailyPrecipitation + }; + if (!supportedTypes.Contains(timeSeriesType)) + throw new ArgumentException( + "BOM API supports DailyDischarge, DailyStage, InstantaneousDischarge, InstantaneousStage, and DailyPrecipitation.", + nameof(timeSeriesType)); // Check connectivity if (!await IsConnectedToInternet()) @@ -764,10 +960,16 @@ public static async Task FromABOM( DateTime sd = startDate ?? new DateTime(1800, 1, 1); DateTime ed = endDate ?? DateTime.Today; - // Determine parameter type based on requested series - string parameterType = timeSeriesType == TimeSeriesType.DailyDischarge - ? "Water Course Discharge" - : "Water Course Level"; + // Determine parameter type and ts_name search pattern based on requested series + bool isDischarge = timeSeriesType == TimeSeriesType.DailyDischarge || timeSeriesType == TimeSeriesType.InstantaneousDischarge; + bool isStage = timeSeriesType == TimeSeriesType.DailyStage || timeSeriesType == TimeSeriesType.InstantaneousStage; + bool isPrecip = timeSeriesType == TimeSeriesType.DailyPrecipitation; + bool isInstantaneous = timeSeriesType == TimeSeriesType.InstantaneousDischarge || timeSeriesType == TimeSeriesType.InstantaneousStage; + + string parameterType; + if (isDischarge) parameterType = "Water Course Discharge"; + else if (isStage) parameterType = "Water Course Level"; + else parameterType = "Rainfall"; // Step 1: Get timeseries list to find the appropriate ts_id string tsListUrl = "http://www.bom.gov.au/waterdata/services" + @@ -787,7 +989,7 @@ public static async Task FromABOM( using (var client = new HttpClient(handler)) { // Add browser-like headers to avoid security proxy issues - client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + client.DefaultRequestHeaders.Add("User-Agent", UserAgent); client.DefaultRequestHeaders.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); @@ -854,22 +1056,33 @@ public static async Task FromABOM( if (tsIdIndex == -1) throw new Exception("Could not find ts_id in response"); - // Look for daily mean time series (prioritize merged/quality-checked data) - for (int i = 1; i < root.GetArrayLength(); i++) - { - var row = root[i]; - string? tsName = tsNameIndex >= 0 ? row[tsNameIndex].GetString() : ""; + // Determine the ts_name pattern to search for. + // Prioritize DMQaQc.Merged (quality-controlled merged data). + string tsNamePattern; + if (isInstantaneous) + tsNamePattern = "AsStored"; + else if (isPrecip) + tsNamePattern = "DailyTotal"; + else + tsNamePattern = "DailyMean"; - if (tsName == null) continue; - // Prioritize: DMQaQc.Merged.DailyMean.24HR or similar daily mean series - if (tsName.Contains("DailyMean") || tsName.Contains("Daily Mean")) + // Two-pass search: first look for DMQaQc.Merged match, then any match + for (int pass = 0; pass < 2 && tsId == null; pass++) + { + for (int i = 1; i < root.GetArrayLength(); i++) { + var row = root[i]; + string tsName = tsNameIndex >= 0 ? row[tsNameIndex].GetString() ?? "" : ""; + + if (!tsName.Contains(tsNamePattern)) continue; + if (pass == 0 && !tsName.StartsWith("DMQaQc.Merged")) continue; + tsId = row[tsIdIndex].GetString(); break; } } - // If no daily mean found, take the first available series + // If no match found, take the first available series if (tsId == null && root.GetArrayLength() > 1) { tsId = root[1][tsIdIndex].GetString(); @@ -887,7 +1100,7 @@ public static async Task FromABOM( $"&from={sd:yyyy-MM-dd}" + $"&to={ed:yyyy-MM-dd}"; - var ts = new TimeSeries(TimeInterval.OneDay); + var ts = isInstantaneous ? new TimeSeries(TimeInterval.Irregular) : new TimeSeries(TimeInterval.OneDay); DateTime? prevDate = null; // Create HttpClientHandler with automatic decompression @@ -899,7 +1112,7 @@ public static async Task FromABOM( using (var client = new HttpClient(handler2)) { // Add browser-like headers to avoid security proxy issues - client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + client.DefaultRequestHeaders.Add("User-Agent", UserAgent); client.DefaultRequestHeaders.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); @@ -960,24 +1173,35 @@ public static async Task FromABOM( double raw = valueElement.GetDouble(); // Apply unit conversion - if (timeSeriesType == TimeSeriesType.DailyDischarge) + if (isDischarge) { // BOM returns discharge in m³/s val = dischargeUnit == DischargeUnit.CubicFeetPerSecond ? raw * 35.3146667 : raw; } - else + else if (isStage) { // BOM returns stage in meters val = heightUnit == HeightUnit.Feet ? raw * 3.280839895 : raw; } + else if (isPrecip) + { + // BOM returns rainfall in millimeters + val = depthUnit switch + { + DepthUnit.Millimeters => raw, + DepthUnit.Centimeters => raw / 10.0, + DepthUnit.Inches => raw / 25.4, + _ => raw + }; + } } - // Fill gaps with NaN - if (prevDate.HasValue && (date - prevDate.Value).Days > 1) + // Fill gaps with NaN for daily series only + if (!isInstantaneous && prevDate.HasValue && (date - prevDate.Value).Days > 1) FillMissingDates(ts, prevDate.Value, date); ts.Add(new SeriesOrdinate(date, val)); diff --git a/Numerics/Data/Time Series/TimeSeries.cs b/Numerics/Data/Time Series/TimeSeries.cs index 53f1bb4d..1a93a993 100644 --- a/Numerics/Data/Time Series/TimeSeries.cs +++ b/Numerics/Data/Time Series/TimeSeries.cs @@ -30,6 +30,7 @@ using Numerics.Data.Statistics; using Numerics.MachineLearning; +using Numerics.Mathematics; using Numerics.Sampling; using System; using System.Collections.Generic; @@ -151,22 +152,15 @@ public TimeSeries(XElement xElement) // Try to parse the invariant date string using TryParseExact // If it fails, do a regular try parse. DateTime index = default; - var ordAttr = ordinate.Attribute("Index"); - if (ordAttr != null) + double value = double.NaN; + var indexAttr = ordinate.Attribute("Index"); + var valueAttr = ordinate.Attribute("Value"); + if (indexAttr != null && !DateTime.TryParseExact(indexAttr.Value, "o", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out index)) { - if (!DateTime.TryParseExact(ordAttr.Value, "o", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out index)) - { - DateTime.TryParse(ordAttr.Value, out index); - } - } - - double value = 0.0; - var ordVal = ordinate.Attribute("Value"); - if (ordVal != null) - { - double.TryParse(ordVal.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out value); + DateTime.TryParse(indexAttr.Value, out index); } - + if (valueAttr != null) + double.TryParse(valueAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out value); Add(new SeriesOrdinate(index, value)); } } @@ -194,7 +188,7 @@ public bool HasMissingValues /// public DateTime StartDate { - get { return _seriesOrdinates.Min(x => x.Index); } + get { return _seriesOrdinates.Count == 0 ? default : _seriesOrdinates.Min(x => x.Index); } } /// @@ -202,7 +196,7 @@ public DateTime StartDate /// public DateTime EndDate { - get { return _seriesOrdinates.Max(x => x.Index); } + get { return _seriesOrdinates.Count == 0 ? default : _seriesOrdinates.Max(x => x.Index); } } /// @@ -336,6 +330,7 @@ public void Multiply(double constant, IList indexes) /// Factor to divide each value by in the series. public void Divide(double constant) { + if (constant == 0) throw new ArgumentException("Cannot divide by zero.", nameof(constant)); SuppressCollectionChanged = true; for (int i = 0; i <= Count - 1; i++) { @@ -514,7 +509,7 @@ public TimeSeries CumulativeSum() double sum = 0d; for (int i = 0; i < Count; i++) { - if (this[i].Value != default && double.IsNaN(this[i].Value) == false) + if (!double.IsNaN(this[i].Value)) sum += this[i].Value; timeSeries.Add(this[i].Clone()); timeSeries.Last().Value = sum; @@ -628,7 +623,7 @@ public void InterpolateMissingData(int maxNumberOfMissing) break; } // the extrapolation case - if (j == Count - 1) + if (j == Count - 1 && i >= 2) { x1 = this[i - 2].Index.ToOADate(); x2 = this[i - 1].Index.ToOADate(); @@ -700,6 +695,14 @@ public void InterpolateMissingData(int maxNumberOfMissing, IList indexes) RaiseCollectionChangedReset(); } + /// + /// Fills missing dates in a time series with a specified value. + /// + /// The time series to fill. + /// The start date of the range. + /// The end date of the range. + /// The value to assign to missing dates. Default is 0.0. + /// A new time series with missing dates filled in. public static TimeSeries FillMissingDates(TimeSeries timeSeries, DateTime startDate, DateTime endDate, double value = 0.0) { if (timeSeries.TimeInterval == TimeInterval.Irregular) throw new Exception("This method does not work with irregular data."); @@ -707,21 +710,22 @@ public static TimeSeries FillMissingDates(TimeSeries timeSeries, DateTime startD // Create a dictionary for fast lookup // var lookup = this.ToList().ToDictionary(p => p.Index, p => p.Value); - var lookup = timeSeries.IndexesToList(); + var lookup = new Dictionary(); + for (int i = 0; i < timeSeries.Count; i++) + lookup[timeSeries[i].Index] = i; var result = new TimeSeries(timeSeries.TimeInterval); // Loop over and add values for (DateTime date = startDate; date <= endDate; date = AddTimeInterval(date, timeSeries.TimeInterval)) { - var idx = lookup.IndexOf(date); - if (idx == -1) + if (lookup.TryGetValue(date, out int idx)) { - result.Add(new SeriesOrdinate(date, value)); + result.Add(new SeriesOrdinate(date, timeSeries[idx].Value)); } else { - result.Add(new SeriesOrdinate(date, timeSeries[idx].Value)); + result.Add(new SeriesOrdinate(date, value)); } } @@ -955,7 +959,7 @@ private bool CheckIfMinStepsExceeded(DateTime startTime, DateTime endTime, int m case TimeInterval.OneQuarter: { - return endTime > startTime.AddYears(1 * minStepsBetweenEvents); + return endTime > startTime.AddMonths(3 * minStepsBetweenEvents); } case TimeInterval.OneYear: @@ -1019,6 +1023,101 @@ public TimeSeries MovingSum(int period) return timeSeries; } + /// + /// Performs classical additive seasonal decomposition using FFT-based seasonal extraction. + /// + /// The seasonal period (e.g., 12 for monthly data with annual seasonality). + /// + /// A tuple containing: + /// + /// Trend: The trend component as a TimeSeries (moving average with the given period). + /// Seasonal: The seasonal component as a double array of length equal to the original series. + /// Residual: The residual component as a TimeSeries (defined where the trend is defined). + /// + /// + /// Thrown when the period is less than 2 or the series contains fewer than 2 complete periods. + public (TimeSeries Trend, double[] Seasonal, TimeSeries Residual) SeasonalDecompose(int period) + { + if (period < 2) + throw new ArgumentException("Period must be at least 2.", nameof(period)); + if (Count < 2 * period) + throw new ArgumentException("Time series must contain at least 2 complete periods.", nameof(period)); + + SortByTime(); + int n = Count; + + // Step 1: Compute trend via moving average + var trend = MovingAverage(period); + + // Step 2: Build full-length arrays and detrend + double[] values = new double[n]; + double[] trendFull = new double[n]; + bool[] hasTrend = new bool[n]; + + for (int i = 0; i < n; i++) + values[i] = this[i].Value; + + // Map trend values to original indices + // MovingAverage outputs (Count - period + 1) values starting at index (period - 1) + int trendStart = period - 1; + for (int i = 0; i < trend.Count; i++) + { + trendFull[trendStart + i] = trend[i].Value; + hasTrend[trendStart + i] = true; + } + + // Create detrended series + double[] detrended = new double[n]; + for (int i = 0; i < n; i++) + detrended[i] = hasTrend[i] ? values[i] - trendFull[i] : 0.0; + + // Step 3: FFT-based seasonal extraction + // Pad to power of 2 for FFT + int fftLength = (int)Math.Pow(2, Math.Ceiling(Math.Log(n, 2))); + if (fftLength < n) fftLength *= 2; + double[] fftData = new double[fftLength]; + Array.Copy(detrended, fftData, n); + + // Forward FFT + Fourier.RealFFT(fftData); + + // Identify harmonic bins of the seasonal frequency + // Harmonic h corresponds to frequency bin k = round(h * fftLength / period) + var harmonicBins = new HashSet(); + for (int h = 1; ; h++) + { + int k = (int)Math.Round((double)h * fftLength / period); + if (k <= 0 || k >= fftLength / 2) break; + harmonicBins.Add(k); + } + + // Keep only harmonic frequencies + double[] filtered = new double[fftLength]; + foreach (int k in harmonicBins) + { + filtered[2 * k] = fftData[2 * k]; + filtered[2 * k + 1] = fftData[2 * k + 1]; + } + + // Inverse FFT + Fourier.RealFFT(filtered, true); + + // Scale by 2/fftLength as required by the RealFFT inverse transform + double[] seasonal = new double[n]; + for (int i = 0; i < n; i++) + seasonal[i] = filtered[i] * 2.0 / fftLength; + + // Step 4: Compute residual + var residual = new TimeSeries(TimeInterval); + for (int i = 0; i < n; i++) + { + if (hasTrend[i]) + residual.Add(new SeriesOrdinate(this[i].Index, values[i] - trendFull[i] - seasonal[i])); + } + + return (trend, seasonal, residual); + } + /// /// Shift all of the dates to match the new start date. /// @@ -1287,9 +1386,20 @@ public double StandardDeviation() { if (Count < 2) return double.NaN; double variance = 0d; - double t = this[0].Value; + // Find first non-NaN value to initialize + double t = 0d; + int startIdx = 0; + for (int i = 0; i < Count; i++) + { + if (!double.IsNaN(this[i].Value)) + { + t = this[i].Value; + startIdx = i + 1; + break; + } + } double n = 1; - for (int i = 1; i < Count; i++) + for (int i = startIdx; i < Count; i++) { if (!double.IsNaN(this[i].Value)) { @@ -1316,7 +1426,7 @@ public double[] SummaryPercentiles() /// A list of k-th percentile values. public double[] Percentiles(IList kValues) { - var data = ValuesToArray(); + var data = ValuesToArray().Where(x => !double.IsNaN(x)).ToArray(); Array.Sort(data); var result = Statistics.Statistics.Percentile(data, kValues, true); return result; @@ -1327,9 +1437,9 @@ public double[] Percentiles(IList kValues) /// public double[,] Duration() { - var result = new double[Count, 2]; - var pp = PlottingPositions.Weibull(Count); - var data = ValuesToArray(); + var data = ValuesToArray().Where(x => !double.IsNaN(x)).ToArray(); + var result = new double[data.Length, 2]; + var pp = PlottingPositions.Weibull(data.Length); Array.Sort(data); Array.Reverse(data); for (int i = 0; i < data.Length; i++) @@ -1479,12 +1589,8 @@ public TimeSeries CalendarYearSeries(BlockFunctionType blockFunction = BlockFunc var result = new TimeSeries(TimeInterval.Irregular); // First, perform smoothing function - TimeSeries? smoothedSeries = null; - if (smoothingFunction == SmoothingFunctionType.None) - { - smoothedSeries = Clone(); - } - else if (smoothingFunction == SmoothingFunctionType.MovingAverage) + TimeSeries smoothedSeries = Clone(); + if (smoothingFunction == SmoothingFunctionType.MovingAverage) { smoothedSeries = period == 1 ? Clone() : MovingAverage(period); } @@ -1498,7 +1604,6 @@ public TimeSeries CalendarYearSeries(BlockFunctionType blockFunction = BlockFunc } // Then, perform block function - if (smoothedSeries == null) return result; for (int i = smoothedSeries.StartDate.Year; i <= smoothedSeries.EndDate.Year; i++) { var blockData = smoothedSeries.Where(x => x.Index.Year == i).ToList(); @@ -1574,12 +1679,8 @@ public TimeSeries CustomYearSeries(int startMonth = 10, BlockFunctionType blockF var result = new TimeSeries(TimeInterval.Irregular); // First, perform smoothing function - TimeSeries? smoothedSeries = null; - if (smoothingFunction == SmoothingFunctionType.None) - { - smoothedSeries = Clone(); - } - else if (smoothingFunction == SmoothingFunctionType.MovingAverage) + TimeSeries smoothedSeries = Clone(); + if (smoothingFunction == SmoothingFunctionType.MovingAverage) { smoothedSeries = period == 1 ? Clone() : MovingAverage(period); } @@ -1593,7 +1694,6 @@ public TimeSeries CustomYearSeries(int startMonth = 10, BlockFunctionType blockF } // Then, shift the dates - if( smoothedSeries == null) return result; int shift = startMonth != 1 ? 12 - startMonth + 1 : 0; smoothedSeries = startMonth != 1 ? smoothedSeries.ShiftDatesByMonth(shift) : smoothedSeries; @@ -1681,12 +1781,8 @@ public TimeSeries CustomYearSeries(int startMonth, int endMonth, BlockFunctionTy var result = new TimeSeries(TimeInterval.Irregular); // First, perform smoothing function - TimeSeries? smoothedSeries = null; - if (smoothingFunction == SmoothingFunctionType.None) - { - smoothedSeries = Clone(); - } - else if (smoothingFunction == SmoothingFunctionType.MovingAverage) + TimeSeries smoothedSeries = Clone(); + if (smoothingFunction == SmoothingFunctionType.MovingAverage) { smoothedSeries = period == 1 ? Clone() : MovingAverage(period); } @@ -1700,7 +1796,6 @@ public TimeSeries CustomYearSeries(int startMonth, int endMonth, BlockFunctionTy } // Then, perform block function - if(smoothedSeries == null) return result; for (int i = smoothedSeries.StartDate.Year; i <= smoothedSeries.EndDate.Year; i++) { @@ -1794,12 +1889,8 @@ public TimeSeries MonthlySeries(BlockFunctionType blockFunction = BlockFunctionT var result = new TimeSeries(TimeInterval.Irregular); // Create smoothed series - TimeSeries? smoothedSeries = null; - if (smoothingFunction == SmoothingFunctionType.None) - { - smoothedSeries = Clone(); - } - else if (smoothingFunction == SmoothingFunctionType.MovingAverage) + TimeSeries smoothedSeries = Clone(); + if (smoothingFunction == SmoothingFunctionType.MovingAverage) { smoothedSeries = period == 1 ? Clone() : MovingAverage(period); } @@ -1812,7 +1903,6 @@ public TimeSeries MonthlySeries(BlockFunctionType blockFunction = BlockFunctionT smoothedSeries = Difference(period); } - if(smoothedSeries == null) return result; for (int i = smoothedSeries.StartDate.Year; i <= smoothedSeries.EndDate.Year; i++) { @@ -1891,12 +1981,8 @@ public TimeSeries QuarterlySeries(BlockFunctionType blockFunction = BlockFunctio var result = new TimeSeries(TimeInterval.Irregular); // Create smoothed series - TimeSeries? smoothedSeries = null; - if (smoothingFunction == SmoothingFunctionType.None) - { - smoothedSeries = Clone(); - } - else if (smoothingFunction == SmoothingFunctionType.MovingAverage) + TimeSeries smoothedSeries = Clone(); + if (smoothingFunction == SmoothingFunctionType.MovingAverage) { smoothedSeries = period == 1 ? Clone() : MovingAverage(period); } @@ -1909,7 +1995,6 @@ public TimeSeries QuarterlySeries(BlockFunctionType blockFunction = BlockFunctio smoothedSeries = Difference(period); } - if (smoothedSeries == null) return result; for (int i = smoothedSeries.StartDate.Year; i <= smoothedSeries.EndDate.Year; i++) { @@ -2004,12 +2089,8 @@ public TimeSeries QuarterlySeries(BlockFunctionType blockFunction = BlockFunctio public TimeSeries PeaksOverThresholdSeries(double threshold, int minStepsBetweenEvents = 1, SmoothingFunctionType smoothingFunction = SmoothingFunctionType.None, int period = 1) { // Create smoothed time series - TimeSeries? smoothedSeries = null; - if (smoothingFunction == SmoothingFunctionType.None) - { - smoothedSeries = Clone(); - } - else if (smoothingFunction == SmoothingFunctionType.MovingAverage) + TimeSeries smoothedSeries = Clone(); + if (smoothingFunction == SmoothingFunctionType.MovingAverage) { smoothedSeries = period == 1 ? Clone() : MovingAverage(period); } @@ -2026,7 +2107,6 @@ public TimeSeries PeaksOverThresholdSeries(double threshold, int minStepsBetween int i = 0, idx, idxMax; var clusters = new List(); - if(smoothedSeries == null) return new TimeSeries(TimeInterval.Irregular); while (i < smoothedSeries.Count) { if (!double.IsNaN(smoothedSeries[i].Value) && smoothedSeries[i].Value > threshold) @@ -2117,7 +2197,8 @@ public TimeSeries ResampleWithKNN(int timeSteps, int k, int seed = 12345) { double val = (currentValue - mean) / stdDev; var kNearest = kNN.GetNeighbors([val]); - int selectedIdx = kNearest[prng.Next(k)]; + if (kNearest == null) continue; + int selectedIdx = kNearest[prng.Next(kNearest.Length)]; currentValue = this[selectedIdx].Value; currentDate = TimeSeries.AddTimeInterval(currentDate, TimeInterval); timeSeries.Add(new SeriesOrdinate(currentDate, currentValue)); diff --git a/Numerics/Distributions/Bivariate Copulas/AMHCopula.cs b/Numerics/Distributions/Bivariate Copulas/AMHCopula.cs index 3b6ab3da..350d0d96 100644 --- a/Numerics/Distributions/Bivariate Copulas/AMHCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/AMHCopula.cs @@ -72,7 +72,7 @@ public AMHCopula(double theta) /// The dependency parameter, θ. ///The X marginal distribution for the copula. ///The Y marginal distribution for the copula. - public AMHCopula(double theta, IUnivariateDistribution marginalDistributionX, IUnivariateDistribution marginalDistributionY) + public AMHCopula(double theta, IUnivariateDistribution? marginalDistributionX, IUnivariateDistribution? marginalDistributionY) { Theta = theta; MarginalDistributionX = marginalDistributionX; @@ -110,7 +110,7 @@ public override double ThetaMaximum } /// - public override ArgumentOutOfRangeException ValidateParameter(double parameter, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameter(double parameter, bool throwException) { if (parameter < ThetaMinimum) { @@ -122,7 +122,7 @@ public override ArgumentOutOfRangeException ValidateParameter(double parameter, if (throwException) throw new ArgumentOutOfRangeException(nameof(Theta), "The dependency parameter θ (theta) must be less than or equal to " + ThetaMaximum.ToString() + "."); return new ArgumentOutOfRangeException(nameof(Theta), "The dependency parameter θ (theta) must be less than or equal to " + ThetaMaximum.ToString() + "."); } - return new ArgumentOutOfRangeException(nameof(Theta), "Parameter is valid."); + return null; } /// @@ -182,6 +182,18 @@ public override double[] InverseCDF(double u, double v) return [u, v]; } + /// + /// Gets the upper tail dependence coefficient λ_U = 0. + /// The AMH copula has no tail dependence. + /// + public override double UpperTailDependence => 0.0; + + /// + /// Gets the lower tail dependence coefficient λ_L = 0. + /// The AMH copula has no tail dependence. + /// + public override double LowerTailDependence => 0.0; + /// public override BivariateCopula Clone() { @@ -212,9 +224,9 @@ public void SetThetaFromTau(IList sampleDataX, IList sampleDataY } /// - public override double[] ParameterConstraints(IList sampleDataX, IList sampleDataY) + public override double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY) { - return [-1 + Tools.DoubleMachineEpsilon, 1 - Tools.DoubleMachineEpsilon]; + return new double[,] { { -1 + Tools.DoubleMachineEpsilon, 1 - Tools.DoubleMachineEpsilon } }; } } diff --git a/Numerics/Distributions/Bivariate Copulas/Base/ArchimedeanCopula.cs b/Numerics/Distributions/Bivariate Copulas/Base/ArchimedeanCopula.cs index 5b23539a..201c02cb 100644 --- a/Numerics/Distributions/Bivariate Copulas/Base/ArchimedeanCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/Base/ArchimedeanCopula.cs @@ -65,7 +65,19 @@ public override string ParameterNameShortForm } /// - public override ArgumentOutOfRangeException ValidateParameter(double parameter, bool throwException) + public override int NumberOfCopulaParameters => 1; + + /// + public override double[] GetCopulaParameters => new double[] { Theta }; + + /// + public override void SetCopulaParameters(double[] parameters) + { + Theta = parameters[0]; + } + + /// + public override ArgumentOutOfRangeException? ValidateParameter(double parameter, bool throwException) { if (parameter < ThetaMinimum) { diff --git a/Numerics/Distributions/Bivariate Copulas/Base/BivariateCopula.cs b/Numerics/Distributions/Bivariate Copulas/Base/BivariateCopula.cs index 60944869..fa5531c9 100644 --- a/Numerics/Distributions/Bivariate Copulas/Base/BivariateCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/Base/BivariateCopula.cs @@ -108,10 +108,10 @@ public bool ParametersValid } /// - public virtual IUnivariateDistribution MarginalDistributionX { get; set; } = null!; + public virtual IUnivariateDistribution? MarginalDistributionX { get; set; } /// - public virtual IUnivariateDistribution MarginalDistributionY { get; set; } = null!; + public virtual IUnivariateDistribution? MarginalDistributionY { get; set; } /// public abstract string DisplayName { get; } @@ -119,6 +119,12 @@ public bool ParametersValid /// public abstract string ShortDisplayName { get; } + /// + public abstract double UpperTailDependence { get; } + + /// + public abstract double LowerTailDependence { get; } + #endregion #region Methods @@ -144,10 +150,19 @@ public double LogPDF(double u, double v) public abstract double[] InverseCDF(double u, double v); /// - public abstract double[] ParameterConstraints(IList sampleDataX, IList sampleDataY); + public abstract int NumberOfCopulaParameters { get; } + + /// + public abstract double[] GetCopulaParameters { get; } + + /// + public abstract void SetCopulaParameters(double[] parameters); + + /// + public abstract double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY); /// - public abstract ArgumentOutOfRangeException ValidateParameter(double parameter, bool throwException); + public abstract ArgumentOutOfRangeException? ValidateParameter(double parameter, bool throwException); /// /// Create a deep copy of the copula. diff --git a/Numerics/Distributions/Bivariate Copulas/Base/BivariateCopulaEstimation.cs b/Numerics/Distributions/Bivariate Copulas/Base/BivariateCopulaEstimation.cs index e2378bd9..21c8e927 100644 --- a/Numerics/Distributions/Bivariate Copulas/Base/BivariateCopulaEstimation.cs +++ b/Numerics/Distributions/Bivariate Copulas/Base/BivariateCopulaEstimation.cs @@ -1,4 +1,4 @@ -/* +/* * NOTICE: * The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about * the results, or appropriateness of outputs, obtained from Numerics. @@ -30,6 +30,7 @@ using Numerics.Data.Statistics; using Numerics.Mathematics.Optimization; +using Numerics.Sampling.MCMC; using System; using System.Collections.Generic; using System.Linq; @@ -44,6 +45,12 @@ namespace Numerics.Distributions.Copulas /// Authors: /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil /// + /// + /// Supports copulas with one or more parameters. For single-parameter copulas, BrentSearch (1D) is used. + /// For multi-parameter copulas (e.g., Student's t with rho and nu), NelderMead is used. + /// Bayesian estimation uses the Adaptive Random Walk Metropolis-Hastings (ARWMH) MCMC sampler + /// with uniform priors on the parameter constraints by default. + /// /// [Serializable] public class BivariateCopulaEstimation @@ -52,10 +59,10 @@ public class BivariateCopulaEstimation /// /// Estimate the bivariate copula. /// - /// /// The copula to estimate. /// The sample data for the X variable. /// The sample data for the Y variable. + /// The estimation method to use. public static void Estimate(ref BivariateCopula copula, IList sampleDataX, IList sampleDataY, CopulaEstimationMethod estimationMethod) { switch (estimationMethod) @@ -73,53 +80,111 @@ public static void Estimate(ref BivariateCopula copula, IList sampleData } /// - /// The maximum pseudo likelihood method. + /// The maximum pseudo likelihood method. Automatically selects BrentSearch for single-parameter + /// copulas or NelderMead for multi-parameter copulas. /// /// The copula to estimate. - /// The sample data for the X variable.When estimating with pseudo likelihood, this should be the plotting positions of the data. - /// The sample data for the Y variable.When estimating with pseudo likelihood, this should be the plotting positions of the data. + /// The sample data for the X variable. When estimating with pseudo likelihood, this should be the plotting positions of the data. + /// The sample data for the Y variable. When estimating with pseudo likelihood, this should be the plotting positions of the data. private static void MPL(BivariateCopula copula, IList sampleDataX, IList sampleDataY) { - // Get constraints - var LU = copula.ParameterConstraints(sampleDataX, sampleDataY); + var constraints = copula.ParameterConstraints(sampleDataX, sampleDataY); + int nParams = copula.NumberOfCopulaParameters; - // Solve using Brent method - Func func = (x) => + if (nParams == 1) { - var C = copula.Clone(); - C.Theta = x; - return C.PseudoLogLikelihood(sampleDataX, sampleDataY); - }; - var brent = new BrentSearch(func, LU[0], LU[1]); - brent.Maximize(); - copula.Theta = brent.BestParameterSet.Values[0]; + // Use BrentSearch for 1D optimization + Func func = (x) => + { + var C = copula.Clone(); + C.SetCopulaParameters(new double[] { x }); + return C.PseudoLogLikelihood(sampleDataX, sampleDataY); + }; + var brent = new BrentSearch(func, constraints[0, 0], constraints[0, 1]); + brent.Maximize(); + copula.SetCopulaParameters(new double[] { brent.BestParameterSet.Values[0] }); + } + else + { + // Use NelderMead for multi-dimensional optimization + var initials = copula.GetCopulaParameters; + var lowers = new double[nParams]; + var uppers = new double[nParams]; + for (int i = 0; i < nParams; i++) + { + lowers[i] = constraints[i, 0]; + uppers[i] = constraints[i, 1]; + // Clamp initials to be within bounds + initials[i] = Math.Max(lowers[i], Math.Min(uppers[i], initials[i])); + } + + Func func = (x) => + { + var C = copula.Clone(); + C.SetCopulaParameters(x); + return C.PseudoLogLikelihood(sampleDataX, sampleDataY); + }; + + var solver = new NelderMead(func, nParams, initials, lowers, uppers); + solver.Maximize(); + copula.SetCopulaParameters(solver.BestParameterSet.Values); + } } /// - /// The inference from margins method. + /// The inference from margins method. Automatically selects BrentSearch for single-parameter + /// copulas or NelderMead for multi-parameter copulas. /// /// The copula to estimate. /// The sample data for the X variable. /// The sample data for the Y variable. private static void IFM(BivariateCopula copula, IList sampleDataX, IList sampleDataY) { - // Get constraints - var LU = copula.ParameterConstraints(sampleDataX, sampleDataY); + var constraints = copula.ParameterConstraints(sampleDataX, sampleDataY); + int nParams = copula.NumberOfCopulaParameters; - // Solve using Brent method - Func func = (x) => + if (nParams == 1) { - var C = copula.Clone(); - C.Theta = x; - return C.IFMLogLikelihood(sampleDataX, sampleDataY); - }; - var brent = new BrentSearch(func, LU[0], LU[1]); - brent.Maximize(); - copula.Theta = brent.BestParameterSet.Values[0]; + // Use BrentSearch for 1D optimization + Func func = (x) => + { + var C = copula.Clone(); + C.SetCopulaParameters(new double[] { x }); + return C.IFMLogLikelihood(sampleDataX, sampleDataY); + }; + var brent = new BrentSearch(func, constraints[0, 0], constraints[0, 1]); + brent.Maximize(); + copula.SetCopulaParameters(new double[] { brent.BestParameterSet.Values[0] }); + } + else + { + // Use NelderMead for multi-dimensional optimization + var initials = copula.GetCopulaParameters; + var lowers = new double[nParams]; + var uppers = new double[nParams]; + for (int i = 0; i < nParams; i++) + { + lowers[i] = constraints[i, 0]; + uppers[i] = constraints[i, 1]; + initials[i] = Math.Max(lowers[i], Math.Min(uppers[i], initials[i])); + } + + Func func = (x) => + { + var C = copula.Clone(); + C.SetCopulaParameters(x); + return C.IFMLogLikelihood(sampleDataX, sampleDataY); + }; + + var solver = new NelderMead(func, nParams, initials, lowers, uppers); + solver.Maximize(); + copula.SetCopulaParameters(solver.BestParameterSet.Values); + } } /// - /// The maximum likelihood estimation method. + /// The maximum likelihood estimation method. Jointly estimates copula parameters and marginal + /// distribution parameters using NelderMead optimization. /// /// The copula to estimate. /// The sample data for the X variable. @@ -127,77 +192,79 @@ private static void IFM(BivariateCopula copula, IList sampleDataX, IList private static void MLE(BivariateCopula copula, IList sampleDataX, IList sampleDataY) { // See if marginals are estimable - IMaximumLikelihoodEstimation margin1 = (IMaximumLikelihoodEstimation)copula.MarginalDistributionX; - IMaximumLikelihoodEstimation margin2 = (IMaximumLikelihoodEstimation)copula.MarginalDistributionY; - if (margin1 == null || margin2 == null) throw new ArgumentOutOfRangeException("marginal distributions", "There marginal distributions must implement the IMaximumLikelihoodEstimation interface to use this method."); + IMaximumLikelihoodEstimation? margin1 = copula.MarginalDistributionX as IMaximumLikelihoodEstimation; + IMaximumLikelihoodEstimation? margin2 = copula.MarginalDistributionY as IMaximumLikelihoodEstimation; + if (margin1 == null || margin2 == null) throw new ArgumentOutOfRangeException("marginal distributions", "The marginal distributions must implement the IMaximumLikelihoodEstimation interface to use this method."); - int np1 = copula.MarginalDistributionX.NumberOfParameters; - int np2 = copula.MarginalDistributionY.NumberOfParameters; + int nCopula = copula.NumberOfCopulaParameters; + int np1 = copula.MarginalDistributionX!.NumberOfParameters; + int np2 = copula.MarginalDistributionY!.NumberOfParameters; + int totalParams = nCopula + np1 + np2; - // Get constraints - var initials = new double[1 + np1 + np2]; - var lowers = new double[1 + np1 + np2]; - var uppers = new double[1 + np1 + np2]; + var initials = new double[totalParams]; + var lowers = new double[totalParams]; + var uppers = new double[totalParams]; - // Theta - // get ranks of data + // Get ranks and plotting positions for initial MPL estimate var rank1 = Statistics.RanksInPlace(sampleDataX.ToArray()); var rank2 = Statistics.RanksInPlace(sampleDataY.ToArray()); - // get plotting positions for (int i = 0; i < rank1.Length; i++) { rank1[i] = rank1[i] / (rank1.Length + 1d); rank2[i] = rank2[i] / (rank2.Length + 1d); } - // Get constraints - var LU = copula.ParameterConstraints(sampleDataX, sampleDataY); - lowers[0] = LU[0]; - uppers[0] = LU[1]; - // Estimate copula using MPL + // Get copula parameter constraints and initial estimates via MPL + var copulaConstraints = copula.ParameterConstraints(sampleDataX, sampleDataY); MPL(copula, rank1, rank2); - initials[0] = copula.Theta; + var copulaParams = copula.GetCopulaParameters; + for (int i = 0; i < nCopula; i++) + { + initials[i] = copulaParams[i]; + lowers[i] = copulaConstraints[i, 0]; + uppers[i] = copulaConstraints[i, 1]; + } // Estimate marginals - ((IEstimation)copula.MarginalDistributionX).Estimate(sampleDataX, ParameterEstimationMethod.MaximumLikelihood); - ((IEstimation)copula.MarginalDistributionY).Estimate(sampleDataY, ParameterEstimationMethod.MaximumLikelihood); - + ((IEstimation)copula.MarginalDistributionX!).Estimate(sampleDataX, ParameterEstimationMethod.MaximumLikelihood); + ((IEstimation)copula.MarginalDistributionY!).Estimate(sampleDataY, ParameterEstimationMethod.MaximumLikelihood); var con = margin1.GetParameterConstraints(sampleDataX); - var parms = copula.MarginalDistributionX.GetParameters; + var parms = copula.MarginalDistributionX!.GetParameters; for (int i = 0; i < np1; i++) { - initials[i + 1] = parms[i]; - lowers[i + 1] = con.Item2[i]; - uppers[i + 1] = con.Item3[i]; + initials[nCopula + i] = parms[i]; + lowers[nCopula + i] = con.Item2[i]; + uppers[nCopula + i] = con.Item3[i]; } con = margin2.GetParameterConstraints(sampleDataY); - parms = copula.MarginalDistributionY.GetParameters; + parms = copula.MarginalDistributionY!.GetParameters; for (int i = 0; i < np2; i++) { - initials[i + 1 + np1] = parms[i]; - lowers[i + 1 + np1] = con.Item2[i]; - uppers[i + 1 + np1] = con.Item3[i]; + initials[nCopula + np1 + i] = parms[i]; + lowers[nCopula + np1 + i] = con.Item2[i]; + uppers[nCopula + np1 + i] = con.Item3[i]; } // Log-likelihood function - Func logLH = (double[] x) => { - // Set copula + Func logLH = (double[] x) => + { + // Set copula parameters var C = copula.Clone(); - C.Theta = x[0]; + var copulaVals = new double[nCopula]; + Array.Copy(x, 0, copulaVals, 0, nCopula); + C.SetCopulaParameters(copulaVals); - // marginal 1 - var m1 = ((UnivariateDistributionBase)copula.MarginalDistributionX).Clone(); + // Marginal 1 + var m1 = ((UnivariateDistributionBase)copula.MarginalDistributionX!).Clone(); var p1 = new double[np1]; - for (int i = 0; i < np1; i++) - p1[i] = x[i + 1]; + Array.Copy(x, nCopula, p1, 0, np1); m1.SetParameters(p1); - // marginal 2 - var m2 = ((UnivariateDistributionBase)copula.MarginalDistributionY).Clone(); + // Marginal 2 + var m2 = ((UnivariateDistributionBase)copula.MarginalDistributionY!).Clone(); var p2 = new double[np2]; - for (int i = 0; i < np2; i++) - p2[i] = x[i + 1 + np1]; + Array.Copy(x, nCopula + np1, p2, 0, np2); m2.SetParameters(p2); C.MarginalDistributionX = m1; @@ -205,22 +272,23 @@ private static void MLE(BivariateCopula copula, IList sampleDataX, IList return C.LogLikelihood(sampleDataX, sampleDataY); }; - var solver = new NelderMead(logLH, lowers.Length, initials, lowers, uppers); + var solver = new NelderMead(logLH, totalParams, initials, lowers, uppers); solver.Maximize(); - // Set parameters for copula and marginals - copula.Theta = solver.BestParameterSet.Values[0]; + // Set copula parameters + var bestCopula = new double[nCopula]; + Array.Copy(solver.BestParameterSet.Values, 0, bestCopula, 0, nCopula); + copula.SetCopulaParameters(bestCopula); - var par = new double[np1]; - for (int i = 0; i < np1; i++) - par[i] = solver.BestParameterSet.Values[i + 1]; - copula.MarginalDistributionX.SetParameters(par); - - par = new double[np1]; - for (int i = 0; i < np2; i++) - par[i] = solver.BestParameterSet.Values[i + 1 + np1]; - copula.MarginalDistributionY.SetParameters(par); + // Set marginal 1 parameters + var par1 = new double[np1]; + Array.Copy(solver.BestParameterSet.Values, nCopula, par1, 0, np1); + copula.MarginalDistributionX!.SetParameters(par1); + // Set marginal 2 parameters + var par2 = new double[np2]; + Array.Copy(solver.BestParameterSet.Values, nCopula + np1, par2, 0, np2); + copula.MarginalDistributionY!.SetParameters(par2); } } diff --git a/Numerics/Distributions/Bivariate Copulas/Base/CopulaEstimationMethod.cs b/Numerics/Distributions/Bivariate Copulas/Base/CopulaEstimationMethod.cs index 41095076..f9b88bf9 100644 --- a/Numerics/Distributions/Bivariate Copulas/Base/CopulaEstimationMethod.cs +++ b/Numerics/Distributions/Bivariate Copulas/Base/CopulaEstimationMethod.cs @@ -55,12 +55,12 @@ public enum CopulaEstimationMethod PseudoLikelihood, /// - /// The inference from margins (IFM) method includes two procedures: + /// The inference from margins (IFM) method includes two procedures: /// 1) marginal distributions are independently estimated from the observed values; - /// 2) the copula dependency is estimate through the maximization of the likelihood function - /// given the marginal distributions. + /// 2) the copula dependency is estimate through the maximization of the likelihood function + /// given the marginal distributions. /// - InferenceFromMargins - + InferenceFromMargins, + } } \ No newline at end of file diff --git a/Numerics/Distributions/Bivariate Copulas/Base/CopulaType.cs b/Numerics/Distributions/Bivariate Copulas/Base/CopulaType.cs index ac0afda7..e08ab51b 100644 --- a/Numerics/Distributions/Bivariate Copulas/Base/CopulaType.cs +++ b/Numerics/Distributions/Bivariate Copulas/Base/CopulaType.cs @@ -65,6 +65,10 @@ public enum CopulaType /// /// Normal /// - Normal + Normal, + /// + /// Student's t + /// + StudentT } } \ No newline at end of file diff --git a/Numerics/Distributions/Bivariate Copulas/Base/IBivariateCopula.cs b/Numerics/Distributions/Bivariate Copulas/Base/IBivariateCopula.cs index de2b2f9f..ff9eb5ff 100644 --- a/Numerics/Distributions/Bivariate Copulas/Base/IBivariateCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/Base/IBivariateCopula.cs @@ -75,12 +75,28 @@ public interface IBivariateCopula : IDistribution /// /// The X marginal distribution for the copula. /// - IUnivariateDistribution MarginalDistributionX { get; set; } + IUnivariateDistribution? MarginalDistributionX { get; set; } /// - /// The Y marginal distribution for the copula. + /// The Y marginal distribution for the copula. /// - IUnivariateDistribution MarginalDistributionY { get; set; } + IUnivariateDistribution? MarginalDistributionY { get; set; } + + /// + /// Gets the number of copula-intrinsic parameters (e.g., 1 for Clayton, 2 for Student's t). + /// + int NumberOfCopulaParameters { get; } + + /// + /// Gets all copula parameters as a double array. + /// + double[] GetCopulaParameters { get; } + + /// + /// Sets all copula parameters from a double array. + /// + /// The parameter values to set. + void SetCopulaParameters(double[] parameters); /// /// Test to see if distribution parameters are valid. @@ -88,14 +104,15 @@ public interface IBivariateCopula : IDistribution /// Dependency parameter. /// Boolean indicating whether to throw the exception or not. /// Nothing if the parameters are valid and the exception if invalid parameters were found. - ArgumentOutOfRangeException ValidateParameter(double parameter, bool throwException); + ArgumentOutOfRangeException? ValidateParameter(double parameter, bool throwException); /// - /// Returns the parameter constraints for the dependency parameter given the data samples. + /// Returns the parameter constraints for all copula parameters given the data samples. + /// Returns a 2D array with shape [NumberOfCopulaParameters, 2] where column 0 is the lower bound and column 1 is the upper bound. /// /// The sample data for the X variable. /// The sample data for the Y variable. - double[] ParameterConstraints(IList sampleDataX, IList sampleDataY); + double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY); /// /// The probability density function (PDF) of the copula evaluated at reduced variates u and v. @@ -125,6 +142,16 @@ public interface IBivariateCopula : IDistribution /// Probability between 0 and 1. double[] InverseCDF(double u, double v); + /// + /// Gets the upper tail dependence coefficient λ_U. + /// + double UpperTailDependence { get; } + + /// + /// Gets the lower tail dependence coefficient λ_L. + /// + double LowerTailDependence { get; } + /// /// Generate random values of a distribution given a sample size. /// diff --git a/Numerics/Distributions/Bivariate Copulas/ClaytonCopula.cs b/Numerics/Distributions/Bivariate Copulas/ClaytonCopula.cs index 0ce2af1a..b5efa774 100644 --- a/Numerics/Distributions/Bivariate Copulas/ClaytonCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/ClaytonCopula.cs @@ -71,7 +71,7 @@ public ClaytonCopula(double theta) /// The dependency parameter, θ. ///The X marginal distribution for the copula. ///The Y marginal distribution for the copula. - public ClaytonCopula(double theta, IUnivariateDistribution marginalDistributionX, IUnivariateDistribution marginalDistributionY) + public ClaytonCopula(double theta, IUnivariateDistribution? marginalDistributionX, IUnivariateDistribution? marginalDistributionY) { Theta = theta; MarginalDistributionX = marginalDistributionX; @@ -160,6 +160,24 @@ public override double[] InverseCDF(double u, double v) return [u, v]; } + /// + /// Gets the upper tail dependence coefficient λ_U = 0. + /// The Clayton copula has no upper tail dependence. + /// + public override double UpperTailDependence => 0.0; + + /// + /// Gets the lower tail dependence coefficient λ_L = 2^(-1/θ). + /// + public override double LowerTailDependence + { + get + { + if (Theta <= 0.0) return 0.0; + return Math.Pow(2.0, -1.0 / Theta); + } + } + /// public override BivariateCopula Clone() { @@ -178,9 +196,9 @@ public void SetThetaFromTau(IList sampleDataX, IList sampleDataY } /// - public override double[] ParameterConstraints(IList sampleDataX, IList sampleDataY) + public override double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY) { - return [-1, 100]; + return new double[,] { { -1, 100 } }; } } diff --git a/Numerics/Distributions/Bivariate Copulas/FrankCopula.cs b/Numerics/Distributions/Bivariate Copulas/FrankCopula.cs index 0b6a06a6..bdee363f 100644 --- a/Numerics/Distributions/Bivariate Copulas/FrankCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/FrankCopula.cs @@ -71,7 +71,7 @@ public FrankCopula(double theta) /// The dependency parameter, θ. ///The X marginal distribution for the copula. ///The Y marginal distribution for the copula. - public FrankCopula(double theta, IUnivariateDistribution marginalDistributionX, IUnivariateDistribution marginalDistributionY) + public FrankCopula(double theta, IUnivariateDistribution? marginalDistributionX, IUnivariateDistribution? marginalDistributionY) { Theta = theta; MarginalDistributionX = marginalDistributionX; @@ -109,7 +109,7 @@ public override double ThetaMaximum } /// - public override ArgumentOutOfRangeException ValidateParameter(double parameter, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameter(double parameter, bool throwException) { if (parameter < ThetaMinimum) { @@ -121,12 +121,13 @@ public override ArgumentOutOfRangeException ValidateParameter(double parameter, if (throwException) throw new ArgumentOutOfRangeException(nameof(Theta), "The dependency parameter θ (theta) must be less than or equal to " + ThetaMaximum.ToString() + "."); return new ArgumentOutOfRangeException(nameof(Theta), "The dependency parameter θ (theta) must be less than or equal to " + ThetaMaximum.ToString() + "."); } - return new ArgumentOutOfRangeException(nameof(Theta),"Parameter is valid."); + return null; } /// public override double Generator(double t) { + if (Math.Abs(Theta) < 1e-10) return t; return -Math.Log((Math.Exp(-Theta * t) - 1d) / (Math.Exp(-Theta) - 1d)); } @@ -183,6 +184,18 @@ public override double[] InverseCDF(double u, double v) return [u, v]; } + /// + /// Gets the upper tail dependence coefficient λ_U = 0. + /// The Frank copula has no tail dependence. + /// + public override double UpperTailDependence => 0.0; + + /// + /// Gets the lower tail dependence coefficient λ_L = 0. + /// The Frank copula has no tail dependence. + /// + public override double LowerTailDependence => 0.0; + /// public override BivariateCopula Clone() { @@ -190,12 +203,12 @@ public override BivariateCopula Clone() } /// - public override double[] ParameterConstraints(IList sampleDataX, IList sampleDataY) + public override double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY) { var tau = Correlation.KendallsTau(sampleDataX, sampleDataY); double L = tau > 0 ? 0.001d : -100d; double U = tau > 0 ? 100d : -0.001d; - return [L, U]; + return new double[,] { { L, U } }; } } diff --git a/Numerics/Distributions/Bivariate Copulas/GumbelCopula.cs b/Numerics/Distributions/Bivariate Copulas/GumbelCopula.cs index aa7ce293..ab719171 100644 --- a/Numerics/Distributions/Bivariate Copulas/GumbelCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/GumbelCopula.cs @@ -72,7 +72,7 @@ public GumbelCopula(double theta) /// The dependency parameter, θ. ///The X marginal distribution for the copula. ///The Y marginal distribution for the copula. - public GumbelCopula(double theta, IUnivariateDistribution marginalDistributionX, IUnivariateDistribution marginalDistributionY) + public GumbelCopula(double theta, IUnivariateDistribution? marginalDistributionX, IUnivariateDistribution? marginalDistributionY) { Theta = theta; MarginalDistributionX = marginalDistributionX; @@ -159,6 +159,23 @@ public override double[] InverseCDF(double u, double v) return [u, v]; } + /// + /// Gets the upper tail dependence coefficient λ_U = 2 - 2^(1/θ). + /// + public override double UpperTailDependence + { + get + { + return 2.0 - Math.Pow(2.0, 1.0 / Theta); + } + } + + /// + /// Gets the lower tail dependence coefficient λ_L = 0. + /// The Gumbel copula has no lower tail dependence. + /// + public override double LowerTailDependence => 0.0; + /// public override BivariateCopula Clone() { @@ -177,9 +194,9 @@ public void SetThetaFromTau(IList sampleDataX, IList sampleDataY } /// - public override double[] ParameterConstraints(IList sampleDataX, IList sampleDataY) + public override double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY) { - return [1, 100]; + return new double[,] { { 1, 100 } }; } } diff --git a/Numerics/Distributions/Bivariate Copulas/JoeCopula.cs b/Numerics/Distributions/Bivariate Copulas/JoeCopula.cs index f469a388..921c86c9 100644 --- a/Numerics/Distributions/Bivariate Copulas/JoeCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/JoeCopula.cs @@ -71,7 +71,7 @@ public JoeCopula(double theta) /// The dependency parameter, θ. ///The X marginal distribution for the copula. ///The Y marginal distribution for the copula. - public JoeCopula(double theta, IUnivariateDistribution marginalDistributionX, IUnivariateDistribution marginalDistributionY) + public JoeCopula(double theta, IUnivariateDistribution? marginalDistributionX, IUnivariateDistribution? marginalDistributionY) { Theta = theta; MarginalDistributionX = marginalDistributionX; @@ -161,6 +161,23 @@ public override double[] InverseCDF(double u, double v) return [u, v]; } + /// + /// Gets the upper tail dependence coefficient λ_U = 2 - 2^(1/θ). + /// + public override double UpperTailDependence + { + get + { + return 2.0 - Math.Pow(2.0, 1.0 / Theta); + } + } + + /// + /// Gets the lower tail dependence coefficient λ_L = 0. + /// The Joe copula has no lower tail dependence. + /// + public override double LowerTailDependence => 0.0; + /// public override BivariateCopula Clone() { @@ -168,9 +185,9 @@ public override BivariateCopula Clone() } /// - public override double[] ParameterConstraints(IList sampleDataX, IList sampleDataY) + public override double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY) { - return [1, 100]; + return new double[,] { { 1, 100 } }; } } } \ No newline at end of file diff --git a/Numerics/Distributions/Bivariate Copulas/NormalCopula.cs b/Numerics/Distributions/Bivariate Copulas/NormalCopula.cs index 7215ba75..37cb620e 100644 --- a/Numerics/Distributions/Bivariate Copulas/NormalCopula.cs +++ b/Numerics/Distributions/Bivariate Copulas/NormalCopula.cs @@ -69,7 +69,7 @@ public NormalCopula(double rho) /// The dependency parameter, θ. ///The X marginal distribution for the copula. ///The Y marginal distribution for the copula. - public NormalCopula(double rho, IUnivariateDistribution marginalDistributionX, IUnivariateDistribution marginalDistributionY) + public NormalCopula(double rho, IUnivariateDistribution? marginalDistributionX, IUnivariateDistribution? marginalDistributionY) { Theta = rho; MarginalDistributionX = marginalDistributionX; @@ -125,7 +125,7 @@ public override double ThetaMaximum } /// - public override ArgumentOutOfRangeException ValidateParameter(double parameter, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameter(double parameter, bool throwException) { if (parameter < ThetaMinimum) { @@ -137,13 +137,25 @@ public override ArgumentOutOfRangeException ValidateParameter(double parameter, if (throwException) throw new ArgumentOutOfRangeException(nameof(Theta), "The correlation parameter ρ (rho) must be less than " + ThetaMaximum.ToString() + "."); return new ArgumentOutOfRangeException(nameof(Theta), "The correlation parameter ρ (rho) must be less than " + ThetaMaximum.ToString() + "."); } - return new ArgumentOutOfRangeException(nameof(Theta),"The parameter is valid."); + return null; } /// - public override double[] ParameterConstraints(IList sampleDataX, IList sampleDataY) + public override int NumberOfCopulaParameters => 1; + + /// + public override double[] GetCopulaParameters => new double[] { Theta }; + + /// + public override void SetCopulaParameters(double[] parameters) + { + Theta = parameters[0]; + } + + /// + public override double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY) { - return [-1 + Tools.DoubleMachineEpsilon, 1 - Tools.DoubleMachineEpsilon]; + return new double[,] { { -1 + Tools.DoubleMachineEpsilon, 1 - Tools.DoubleMachineEpsilon } }; } /// @@ -162,7 +174,10 @@ public override double CDF(double u, double v) { // Validate parameters if (_parametersValid == false) ValidateParameter(Theta, true); - return MultivariateNormal.BivariateCDF(Normal.StandardZ(1 - u), Normal.StandardZ(1 - v), _theta); + // BivariateCDF implements Genz's BVND which computes Phi2(-h,-k;r). + // To get the copula C(u,v) = Phi2(Phi^-1(u), Phi^-1(v); r), + // we pass -Phi^-1(u) = Phi^-1(1-u) as arguments. + return MultivariateNormal.BivariateCDF(-Normal.StandardZ(u), -Normal.StandardZ(v), _theta); } /// @@ -178,6 +193,18 @@ public override double[] InverseCDF(double u, double v) return [u, v]; } + /// + /// Gets the upper tail dependence coefficient λ_U = 0. + /// The Normal copula has no upper tail dependence. + /// + public override double UpperTailDependence => 0.0; + + /// + /// Gets the lower tail dependence coefficient λ_L = 0. + /// The Normal copula has no lower tail dependence. + /// + public override double LowerTailDependence => 0.0; + /// public override BivariateCopula Clone() { diff --git a/Numerics/Distributions/Bivariate Copulas/StudentTCopula.cs b/Numerics/Distributions/Bivariate Copulas/StudentTCopula.cs new file mode 100644 index 00000000..ea8c4f7f --- /dev/null +++ b/Numerics/Distributions/Bivariate Copulas/StudentTCopula.cs @@ -0,0 +1,409 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Collections.Generic; +using Numerics.Mathematics.SpecialFunctions; + +namespace Numerics.Distributions.Copulas +{ + + /// + /// The bivariate Student's t-copula. + /// + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// Description: + /// + /// + /// The Student's t-copula is an elliptical copula derived from the bivariate Student's t-distribution. + /// Unlike the Normal (Gaussian) copula which has zero tail dependence, the t-copula exhibits symmetric + /// upper and lower tail dependence controlled by the degrees of freedom parameter. As the degrees of + /// freedom increase to infinity, the t-copula converges to the Normal copula. + /// + /// + /// The copula is parameterized by the correlation coefficient rho (stored in Theta) and the degrees + /// of freedom nu. The copula density is: + /// + /// c(u,v) = f₂(t⁻¹_ν(u), t⁻¹_ν(v); ρ, ν) / [f_ν(t⁻¹_ν(u)) · f_ν(t⁻¹_ν(v))] + /// + /// where f₂ is the bivariate t density with correlation ρ and ν degrees of freedom, + /// f_ν is the univariate standard t density, and t⁻¹_ν is the univariate t quantile function. + /// + /// + /// Key applications include: + /// + /// Joint flood risk analysis in TotalRisk where tail dependence between flood variables is critical. + /// Bivariate frequency analysis in RMC-BestFit for modeling dependent extremes. + /// Hydrologic risk assessment where normal copulas underestimate joint extreme event probabilities. + /// + /// + /// + /// References: + /// + /// + /// + /// + /// Demarta, S. and McNeil, A.J. (2005). "The t Copula and Related Copulas." + /// International Statistical Review, 73(1), 111-129. + /// + /// + /// Nelsen, R.B. (2006). "An Introduction to Copulas." 2nd ed. Springer. Chapter 4. + /// + /// + /// + /// + /// + /// + /// + [Serializable] + public class StudentTCopula : BivariateCopula + { + + /// + /// Constructs a bivariate Student's t-copula with default parameters: ρ = 0 and ν = 5. + /// + public StudentTCopula() + { + _nu = 5; + Theta = 0.0d; + } + + /// + /// Constructs a bivariate Student's t-copula with specified correlation and degrees of freedom. + /// + /// The correlation parameter ρ (rho). Must be in [-1, 1]. + /// The degrees of freedom ν (nu). Must be greater than 2. + /// Thrown when ν ≤ 2. + public StudentTCopula(double rho, int degreesOfFreedom) + { + if (degreesOfFreedom <= 2) + throw new ArgumentOutOfRangeException(nameof(degreesOfFreedom), "The degrees of freedom must be greater than 2."); + _nu = degreesOfFreedom; + Theta = rho; + } + + /// + /// Constructs a bivariate Student's t-copula with specified parameters and marginal distributions. + /// + /// The correlation parameter ρ (rho). Must be in [-1, 1]. + /// The degrees of freedom ν (nu). Must be greater than 2. + /// The X marginal distribution. + /// The Y marginal distribution. + /// Thrown when ν ≤ 2. + public StudentTCopula(double rho, int degreesOfFreedom, IUnivariateDistribution? marginalDistributionX, IUnivariateDistribution? marginalDistributionY) + { + if (degreesOfFreedom <= 2) + throw new ArgumentOutOfRangeException(nameof(degreesOfFreedom), "The degrees of freedom must be greater than 2."); + _nu = degreesOfFreedom; + Theta = rho; + MarginalDistributionX = marginalDistributionX; + MarginalDistributionY = marginalDistributionY; + } + + private int _nu; + + /// + /// Gets or sets the degrees of freedom ν (nu). Must be greater than 2. + /// + /// Thrown when the value is ≤ 2. + public int DegreesOfFreedom + { + get { return _nu; } + set + { + if (value <= 2) + throw new ArgumentOutOfRangeException(nameof(DegreesOfFreedom), "The degrees of freedom must be greater than 2."); + _nu = value; + } + } + + /// + public override CopulaType Type + { + get { return CopulaType.StudentT; } + } + + /// + public override string DisplayName + { + get { return "Student's t"; } + } + + /// + public override string ShortDisplayName + { + get { return "t"; } + } + + /// + public override string[,] ParameterToString + { + get + { + var parmString = new string[2, 2]; + parmString[0, 0] = "Correlation (ρ)"; + parmString[0, 1] = Theta.ToString(); + parmString[1, 0] = "Degrees of Freedom (ν)"; + parmString[1, 1] = _nu.ToString(); + return parmString; + } + } + + /// + public override string ParameterNameShortForm + { + get { return "ρ"; } + } + + /// + public override double ThetaMinimum + { + get { return -1.0d; } + } + + /// + public override double ThetaMaximum + { + get { return 1.0d; } + } + + /// + public override ArgumentOutOfRangeException? ValidateParameter(double parameter, bool throwException) + { + if (parameter < ThetaMinimum) + { + if (throwException) throw new ArgumentOutOfRangeException(nameof(Theta), "The correlation parameter ρ (rho) must be greater than " + ThetaMinimum.ToString() + "."); + return new ArgumentOutOfRangeException(nameof(Theta), "The correlation parameter ρ (rho) must be greater than " + ThetaMinimum.ToString() + "."); + } + if (parameter > ThetaMaximum) + { + if (throwException) throw new ArgumentOutOfRangeException(nameof(Theta), "The correlation parameter ρ (rho) must be less than " + ThetaMaximum.ToString() + "."); + return new ArgumentOutOfRangeException(nameof(Theta), "The correlation parameter ρ (rho) must be less than " + ThetaMaximum.ToString() + "."); + } + return null; + } + + /// + public override int NumberOfCopulaParameters => 2; + + /// + public override double[] GetCopulaParameters => new double[] { Theta, (double)DegreesOfFreedom }; + + /// + public override void SetCopulaParameters(double[] parameters) + { + Theta = parameters[0]; + DegreesOfFreedom = Math.Max(3, (int)Math.Round(parameters[1])); + } + + /// + public override double[,] ParameterConstraints(IList sampleDataX, IList sampleDataY) + { + return new double[,] + { + { -1 + Tools.DoubleMachineEpsilon, 1 - Tools.DoubleMachineEpsilon }, + { 3, 60 } + }; + } + + /// + /// Returns the copula probability density function (PDF) evaluated at (u, v). + /// + /// The reduced variate between 0 and 1. + /// The reduced variate between 0 and 1. + /// The copula density c(u, v). + /// + /// + /// The t-copula density is computed in log-space for numerical stability: + /// + /// log c = LogΓ((ν+2)/2) + LogΓ(ν/2) - 2·LogΓ((ν+1)/2) + /// - ½·log(1-ρ²) + /// - ((ν+2)/2)·log(1 + Q/(ν(1-ρ²))) + /// + ((ν+1)/2)·log(1 + x₁²/ν) + ((ν+1)/2)·log(1 + x₂²/ν) + /// + /// where x₁ = t⁻¹_ν(u), x₂ = t⁻¹_ν(v), and Q = x₁² - 2ρx₁x₂ + x₂². + /// + /// + public override double PDF(double u, double v) + { + // Validate parameters + if (_parametersValid == false) ValidateParameter(Theta, true); + + double r = _theta; + double nu = _nu; + + // Transform to t-quantiles + var tDist = new StudentT(0, 1, _nu); + double x1 = tDist.InverseCDF(u); + double x2 = tDist.InverseCDF(v); + + // Compute log copula density in log-space for numerical stability + double r2 = r * r; + double logC = Gamma.LogGamma((nu + 2.0) / 2.0) + + Gamma.LogGamma(nu / 2.0) + - 2.0 * Gamma.LogGamma((nu + 1.0) / 2.0) + - 0.5 * Math.Log(1.0 - r2); + + // Quadratic form: Q = (x1^2 - 2*rho*x1*x2 + x2^2) + double Q = x1 * x1 - 2.0 * r * x1 * x2 + x2 * x2; + logC -= ((nu + 2.0) / 2.0) * Math.Log(1.0 + Q / (nu * (1.0 - r2))); + logC += ((nu + 1.0) / 2.0) * Math.Log(1.0 + x1 * x1 / nu); + logC += ((nu + 1.0) / 2.0) * Math.Log(1.0 + x2 * x2 / nu); + + return Math.Exp(logC); + } + + /// + /// Returns the copula cumulative distribution function (CDF) evaluated at (u, v). + /// + /// The reduced variate between 0 and 1. + /// The reduced variate between 0 and 1. + /// The copula CDF C(u, v). + /// + /// + /// The t-copula CDF is computed using the bivariate Student's t CDF: + /// + /// C(u, v) = F₂(t⁻¹_ν(u), t⁻¹_ν(v); ρ, ν) + /// + /// where F₂ is the bivariate Student's t CDF evaluated via the class. + /// + /// + public override double CDF(double u, double v) + { + // Validate parameters + if (_parametersValid == false) ValidateParameter(Theta, true); + + double r = _theta; + + // Transform to t-quantiles + var tDist = new StudentT(0, 1, _nu); + double x1 = tDist.InverseCDF(u); + double x2 = tDist.InverseCDF(v); + + // Evaluate bivariate t CDF + var scaleMatrix = new double[,] { { 1.0, r }, { r, 1.0 } }; + var mvt = new MultivariateStudentT(_nu, new double[] { 0.0, 0.0 }, scaleMatrix); + return mvt.CDF(new double[] { x1, x2 }); + } + + /// + /// Returns the inverse CDF (conditional sampling) for the copula. + /// + /// The first uniform variate in (0, 1). + /// The second uniform variate in (0, 1), used as the conditional probability. + /// + /// A 2-element array [u, v'] where v' is the conditionally sampled variate. + /// + /// + /// + /// Uses the conditional distribution of the bivariate Student's t: + /// X₂ | X₁ = x₁ ~ t_{ν+1}(ρ·x₁, √((1-ρ²)(ν+x₁²)/(ν+1))) + /// + /// + /// The algorithm is: + /// + /// Transform u to t-quantile: x₁ = t⁻¹_ν(u) + /// Sample from the conditional t_{ν+1} distribution using v + /// Transform the conditional sample back to uniform: v' = t_ν(x₂) + /// + /// + /// + public override double[] InverseCDF(double u, double v) + { + // Validate parameters + if (_parametersValid == false) ValidateParameter(Theta, true); + + double r = _theta; + double nu = _nu; + + // Transform u to t-quantile + var tNu = new StudentT(0, 1, _nu); + double x1 = tNu.InverseCDF(u); + + // Conditional distribution: X2|X1=x1 ~ t_{ν+1} with location = ρ·x1, scale = √((1-ρ²)(ν+x1²)/(ν+1)) + var tNu1 = new StudentT(0, 1, _nu + 1); + double z2 = tNu1.InverseCDF(v); + + double conditionalScale = Math.Sqrt((1.0 - r * r) * (nu + x1 * x1) / (nu + 1.0)); + double x2 = r * x1 + conditionalScale * z2; + + // Transform back to uniform + v = tNu.CDF(x2); + return [u, v]; + } + + /// + /// Gets the upper tail dependence coefficient λ_U. + /// + /// + /// + /// The t-copula has symmetric upper and lower tail dependence: + /// + /// λ_U = λ_L = 2 · t_{ν+1}(-√((ν+1)(1-ρ)/(1+ρ))) + /// + /// where t_{ν+1} is the CDF of the univariate Student's t with ν+1 degrees of freedom. + /// For ρ = -1, λ = 0. For ρ = 1, λ = 1. As ν → ∞, λ → 0 (Normal copula limit). + /// + /// + public override double UpperTailDependence + { + get + { + double r = _theta; + double nu = _nu; + + if (r >= 1.0) return 1.0; + if (r <= -1.0) return 0.0; + + double arg = -Math.Sqrt((nu + 1.0) * (1.0 - r) / (1.0 + r)); + var tDist = new StudentT(0, 1, _nu + 1); + return 2.0 * tDist.CDF(arg); + } + } + + /// + /// Gets the lower tail dependence coefficient λ_L. + /// The t-copula has symmetric tail dependence, so λ_L = λ_U. + /// + public override double LowerTailDependence => UpperTailDependence; + + /// + public override BivariateCopula Clone() + { + return new StudentTCopula(Theta, _nu, MarginalDistributionX, MarginalDistributionY); + } + + } +} diff --git a/Numerics/Distributions/Multivariate/Base/MultivariateDistributionType.cs b/Numerics/Distributions/Multivariate/Base/MultivariateDistributionType.cs index 9f34fe3f..5ee0a75c 100644 --- a/Numerics/Distributions/Multivariate/Base/MultivariateDistributionType.cs +++ b/Numerics/Distributions/Multivariate/Base/MultivariateDistributionType.cs @@ -52,6 +52,18 @@ public enum MultivariateDistributionType /// /// Multivariate Normal (MVN) distribution. /// - MultivariateNormal + MultivariateNormal, + /// + /// Dirichlet distribution. + /// + Dirichlet, + /// + /// Multinomial distribution. + /// + Multinomial, + /// + /// Multivariate Student's t-distribution. + /// + MultivariateStudentT } } \ No newline at end of file diff --git a/Numerics/Distributions/Multivariate/BivariateEmpirical.cs b/Numerics/Distributions/Multivariate/BivariateEmpirical.cs index 7a6e1d63..59e26fd8 100644 --- a/Numerics/Distributions/Multivariate/BivariateEmpirical.cs +++ b/Numerics/Distributions/Multivariate/BivariateEmpirical.cs @@ -60,9 +60,6 @@ public BivariateEmpirical(Transform x1Transform = Transform.None, Transform x2Tr X1Transform = x1Transform; X2Transform = x2Transform; ProbabilityTransform = probabilityTransform; - X1Values = Array.Empty(); - X2Values = Array.Empty(); - ProbabilityValues = new double[0, 0]; } /// @@ -94,13 +91,13 @@ public BivariateEmpirical(IList x1Values, IList x2Values, double /// Return the array of X1 values (distribution 1). Points On the cumulative curve are specified /// with increasing value and increasing probability. /// - public double[] X1Values { get; private set; } = Array.Empty(); + public double[] X1Values { get; private set; } = null!; /// /// Return the array of X2 values (distribution 2). Points on the cumulative curve are specified /// with increasing value and increasing probability. /// - public double[] X2Values { get; private set; } = Array.Empty(); + public double[] X2Values { get; private set; } = null!; /// /// Return the array of probability values. Points on the cumulative curve are specified diff --git a/Numerics/Distributions/Multivariate/Dirichlet.cs b/Numerics/Distributions/Multivariate/Dirichlet.cs new file mode 100644 index 00000000..30a48090 --- /dev/null +++ b/Numerics/Distributions/Multivariate/Dirichlet.cs @@ -0,0 +1,439 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Linq; +using Numerics.Mathematics.SpecialFunctions; +using Numerics.Sampling; + +namespace Numerics.Distributions +{ + + /// + /// The Dirichlet distribution, a multivariate generalization of the Beta distribution. + /// + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// Description: + /// + /// + /// The Dirichlet distribution Dir(α₁, α₂, ..., αₖ) is a family of continuous multivariate + /// probability distributions parameterized by a vector α of positive real numbers. It is + /// defined on the (K-1)-dimensional simplex, meaning the components x₁, x₂, ..., xₖ satisfy + /// xᵢ > 0 and Σxᵢ = 1. The Dirichlet distribution is the conjugate prior of the categorical + /// and multinomial distributions in Bayesian statistics. + /// + /// + /// Key applications include: + /// + /// Bayesian inference: conjugate prior for mixture model weights in RMC-BestFit. + /// Topic modeling: prior distribution over document-topic proportions. + /// Compositional data analysis: modeling proportions that sum to unity. + /// + /// + /// + /// References: + /// + /// + /// + /// + /// Kotz, S., Balakrishnan, N. and Johnson, N.L. (2000). "Continuous Multivariate Distributions, + /// Volume 1: Models and Applications," 2nd ed. Wiley. Chapter 49. + /// + /// + /// + /// + /// + /// + /// + [Serializable] + public class Dirichlet : MultivariateDistribution + { + + /// + /// Private parameterless constructor for use by Clone(). + /// + private Dirichlet() { } + + /// + /// Constructs a symmetric Dirichlet distribution where all concentration parameters are equal. + /// + /// The number of categories K. Must be at least 2. + /// The common concentration parameter. Must be positive. + /// Thrown when dimension < 2 or alpha <= 0. + public Dirichlet(int dimension, double alpha) + { + if (dimension < 2) + throw new ArgumentOutOfRangeException(nameof(dimension), "The dimension must be at least 2."); + if (double.IsNaN(alpha) || double.IsInfinity(alpha) || alpha <= 0) + throw new ArgumentOutOfRangeException(nameof(alpha), "The concentration parameter must be positive."); + + _alpha = new double[dimension]; + for (int i = 0; i < dimension; i++) + _alpha[i] = alpha; + _dimension = dimension; + ComputeNormalization(); + } + + /// + /// Constructs a Dirichlet distribution with the specified concentration parameter vector. + /// + /// The vector of concentration parameters α₁, α₂, ..., αₖ. Each must be positive. + /// Thrown when any concentration parameter is non-positive, + /// or the vector has fewer than 2 elements. + public Dirichlet(double[] alpha) + { + if (alpha == null || alpha.Length < 2) + throw new ArgumentOutOfRangeException(nameof(alpha), "The concentration parameter vector must have at least 2 elements."); + for (int i = 0; i < alpha.Length; i++) + { + if (double.IsNaN(alpha[i]) || double.IsInfinity(alpha[i]) || alpha[i] <= 0) + throw new ArgumentOutOfRangeException(nameof(alpha), $"Concentration parameter α[{i}] must be positive."); + } + + _alpha = (double[])alpha.Clone(); + _dimension = alpha.Length; + ComputeNormalization(); + } + + private double[] _alpha = null!; + private int _dimension; + private double _logNormalization; // log(B(alpha)) = sum(logGamma(alpha_i)) - logGamma(sum(alpha_i)) + private double _alphaSum; + private double[]? _mean; + private double[]? _variance; + private double[]? _mode; + + /// + /// Computes and caches the normalization constant and alpha sum. + /// + private void ComputeNormalization() + { + _alphaSum = 0; + _logNormalization = 0; + for (int i = 0; i < _dimension; i++) + { + _alphaSum += _alpha[i]; + _logNormalization += Gamma.LogGamma(_alpha[i]); + } + _logNormalization -= Gamma.LogGamma(_alphaSum); + } + + /// + /// Gets the concentration parameter vector α. + /// + public double[] Alpha + { + get { return (double[])_alpha.Clone(); } + } + + /// + /// Gets the sum of all concentration parameters: S = Σαᵢ. + /// + public double AlphaSum + { + get { return _alphaSum; } + } + + /// + public override int Dimension + { + get { return _dimension; } + } + + /// + public override MultivariateDistributionType Type + { + get { return MultivariateDistributionType.Dirichlet; } + } + + /// + public override string DisplayName + { + get { return "Dirichlet"; } + } + + /// + public override string ShortDisplayName + { + get { return "Dir"; } + } + + /// + public override bool ParametersValid + { + get + { + if (_alpha == null || _alpha.Length < 2) return false; + for (int i = 0; i < _alpha.Length; i++) + { + if (double.IsNaN(_alpha[i]) || double.IsInfinity(_alpha[i]) || _alpha[i] <= 0) + return false; + } + return true; + } + } + + /// + /// Gets the mean vector. Mean[i] = αᵢ / S, where S = Σαⱼ. + /// + public double[] Mean + { + get + { + if (_mean == null) + { + _mean = new double[_dimension]; + for (int i = 0; i < _dimension; i++) + _mean[i] = _alpha[i] / _alphaSum; + } + return _mean; + } + } + + /// + /// Gets the marginal variance vector. Var[i] = αᵢ(S - αᵢ) / (S²(S + 1)). + /// + public double[] Variance + { + get + { + if (_variance == null) + { + _variance = new double[_dimension]; + double s2 = _alphaSum * _alphaSum; + double denom = s2 * (_alphaSum + 1.0); + for (int i = 0; i < _dimension; i++) + _variance[i] = _alpha[i] * (_alphaSum - _alpha[i]) / denom; + } + return _variance; + } + } + + /// + /// Gets the mode vector. Mode[i] = (αᵢ - 1) / (S - K) when all αᵢ > 1. + /// + /// Thrown when any αᵢ <= 1, as the mode is not in the interior of the simplex. + public double[] Mode + { + get + { + if (_mode == null) + { + for (int i = 0; i < _dimension; i++) + { + if (_alpha[i] <= 1.0) + throw new InvalidOperationException("The mode is only defined in the interior of the simplex when all αᵢ > 1."); + } + double denom = _alphaSum - _dimension; + _mode = new double[_dimension]; + for (int i = 0; i < _dimension; i++) + _mode[i] = (_alpha[i] - 1.0) / denom; + } + return _mode; + } + } + + /// + /// Gets the covariance between components i and j: Cov(Xᵢ, Xⱼ) = -αᵢαⱼ / (S²(S + 1)). + /// + /// The first component index (0-based). + /// The second component index (0-based). + /// The covariance. Negative for i != j (components are negatively correlated on the simplex). + public double Covariance(int i, int j) + { + if (i < 0 || i >= _dimension || j < 0 || j >= _dimension) + throw new ArgumentOutOfRangeException("Index out of range."); + if (i == j) return Variance[i]; + double denom = _alphaSum * _alphaSum * (_alphaSum + 1.0); + return -_alpha[i] * _alpha[j] / denom; + } + + /// + /// Gets the full covariance matrix. + /// + /// A K x K covariance matrix. + public double[,] CovarianceMatrix() + { + var cov = new double[_dimension, _dimension]; + double denom = _alphaSum * _alphaSum * (_alphaSum + 1.0); + for (int i = 0; i < _dimension; i++) + { + for (int j = 0; j < _dimension; j++) + { + if (i == j) + cov[i, j] = _alpha[i] * (_alphaSum - _alpha[i]) / denom; + else + cov[i, j] = -_alpha[i] * _alpha[j] / denom; + } + } + return cov; + } + + /// + public override double PDF(double[] x) + { + return Math.Exp(LogPDF(x)); + } + + /// + /// Computes the log of the probability density function. + /// + /// A point on the simplex. Components must be positive and sum to 1. + /// + /// The log-density at x. Returns if x is not on the simplex. + /// + /// + /// + /// The log-PDF is computed as: + /// + /// log f(x) = Σ(αᵢ - 1)·log(xᵢ) - log B(α) + /// + /// where B(α) = Π Γ(αᵢ) / Γ(Σαᵢ) is the multivariate Beta function. + /// + /// + public override double LogPDF(double[] x) + { + if (x == null || x.Length != _dimension) return double.MinValue; + + // Check that x is on the simplex + double sum = 0; + for (int i = 0; i < _dimension; i++) + { + if (x[i] <= 0 || x[i] > 1) return double.MinValue; + sum += x[i]; + } + if (Math.Abs(sum - 1.0) > 1e-10) return double.MinValue; + + double logPdf = -_logNormalization; + for (int i = 0; i < _dimension; i++) + logPdf += (_alpha[i] - 1.0) * Math.Log(x[i]); + + return logPdf; + } + + /// + /// The CDF of the Dirichlet distribution is not available in closed form. + /// + /// The vector of x values. + /// This method always throws . + /// Always thrown. The multivariate Dirichlet CDF has no closed-form expression. + public override double CDF(double[] x) + { + throw new NotImplementedException("The CDF of the Dirichlet distribution does not have a closed-form expression."); + } + + /// + public override MultivariateDistribution Clone() + { + var clone = new Dirichlet(); + clone._alpha = (double[])_alpha.Clone(); + clone._dimension = _dimension; + clone._logNormalization = _logNormalization; + clone._alphaSum = _alphaSum; + return clone; + } + + /// + /// Generates random samples from the Dirichlet distribution. + /// + /// The number of samples to generate. + /// Optional seed for reproducibility. Use -1 for a random seed. + /// + /// A 2D array of shape [sampleSize, Dimension]. Each row sums to 1 and lies on the simplex. + /// + /// + /// + /// Sampling uses the gamma distribution representation: draw K independent + /// Gamma(αᵢ, 1) variates yᵢ, then normalize: xᵢ = yᵢ / Σyⱼ. + /// This leverages the existing class. + /// + /// + public double[,] GenerateRandomValues(int sampleSize, int seed = -1) + { + var rng = seed > 0 ? new MersenneTwister(seed) : new MersenneTwister(); + var sample = new double[sampleSize, _dimension]; + + // Create Gamma distributions for each component: Gamma(alpha_i, 1) + // GammaDistribution uses shape-rate parameterization: GammaDistribution(beta=1/scale, alpha=shape) + // Actually, need to check the constructor signature + var gammas = new GammaDistribution[_dimension]; + for (int i = 0; i < _dimension; i++) + gammas[i] = new GammaDistribution(1.0, _alpha[i]); // beta=1 (rate), alpha=shape + + for (int s = 0; s < sampleSize; s++) + { + double sum = 0; + var y = new double[_dimension]; + for (int i = 0; i < _dimension; i++) + { + y[i] = gammas[i].InverseCDF(rng.NextDouble()); + if (y[i] < 0) y[i] = 0; // Guard against numerical issues + sum += y[i]; + } + + // Normalize to simplex + if (sum == 0) + { + for (int i = 0; i < _dimension; i++) + sample[s, i] = 1.0 / _dimension; + continue; + } + for (int i = 0; i < _dimension; i++) + sample[s, i] = y[i] / sum; + } + + return sample; + } + + /// + /// Computes the log of the multivariate Beta function: log B(α) = Σ log Γ(αᵢ) - log Γ(Σαᵢ). + /// + /// The concentration parameter vector. + /// The log of B(α). + public static double LogMultivariateBeta(double[] alpha) + { + double logB = 0; + double sum = 0; + for (int i = 0; i < alpha.Length; i++) + { + logB += Gamma.LogGamma(alpha[i]); + sum += alpha[i]; + } + logB -= Gamma.LogGamma(sum); + return logB; + } + + } +} diff --git a/Numerics/Distributions/Multivariate/Multinomial.cs b/Numerics/Distributions/Multivariate/Multinomial.cs new file mode 100644 index 00000000..8d33674d --- /dev/null +++ b/Numerics/Distributions/Multivariate/Multinomial.cs @@ -0,0 +1,414 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using Numerics.Mathematics.SpecialFunctions; +using Numerics.Sampling; + +namespace Numerics.Distributions +{ + + /// + /// The Multinomial distribution, a generalization of the binomial distribution to K categories. + /// + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// Description: + /// + /// + /// The multinomial distribution models the number of outcomes in K categories from N independent trials, + /// where each trial results in exactly one of the K categories with fixed probabilities p₁, p₂, ..., pₖ. + /// The probability mass function gives the probability of observing x₁ in category 1, x₂ in category 2, + /// etc., where Σxᵢ = N. + /// + /// + /// Key applications include: + /// + /// MCMC sampling: selecting trajectory states in NUTS (No-U-Turn Sampler) via weighted multinomial draws. + /// Categorical data modeling: modeling counts from multiple categories in LifeSim. + /// Bayesian inference: likelihood function for categorical observations. + /// + /// + /// + /// References: + /// + /// + /// + /// + /// Johnson, N.L., Kotz, S. and Balakrishnan, N. (1997). "Discrete Multivariate Distributions." Wiley. + /// + /// + /// + /// + /// + /// + /// + [Serializable] + public class Multinomial : MultivariateDistribution + { + + /// + /// Private parameterless constructor for use by Clone(). + /// + private Multinomial() { } + + /// + /// Constructs a multinomial distribution with the specified number of trials and category probabilities. + /// + /// The number of trials N. Must be positive. + /// The probability vector p₁, ..., pₖ. Each must be in [0, 1] and they must sum to 1. + /// Thrown when N < 1, probabilities have fewer than 2 elements, + /// any probability is negative, or probabilities don't sum to 1. + public Multinomial(int numberOfTrials, double[] probabilities) + { + if (numberOfTrials < 1) + throw new ArgumentOutOfRangeException(nameof(numberOfTrials), "The number of trials must be positive."); + if (probabilities == null || probabilities.Length < 2) + throw new ArgumentOutOfRangeException(nameof(probabilities), "The probability vector must have at least 2 elements."); + + double sum = 0; + for (int i = 0; i < probabilities.Length; i++) + { + if (double.IsNaN(probabilities[i]) || probabilities[i] < 0 || probabilities[i] > 1) + throw new ArgumentOutOfRangeException(nameof(probabilities), $"Probability p[{i}] must be in [0, 1]."); + sum += probabilities[i]; + } + if (Math.Abs(sum - 1.0) > 1e-10) + throw new ArgumentOutOfRangeException(nameof(probabilities), "Probabilities must sum to 1."); + + _n = numberOfTrials; + _p = (double[])probabilities.Clone(); + _dimension = probabilities.Length; + } + + private int _n; + private double[] _p = null!; + private int _dimension; + + /// + /// Gets the number of trials N. + /// + public int NumberOfTrials + { + get { return _n; } + } + + /// + /// Gets the probability vector p. + /// + public double[] Probabilities + { + get { return (double[])_p.Clone(); } + } + + /// + public override int Dimension + { + get { return _dimension; } + } + + /// + public override MultivariateDistributionType Type + { + get { return MultivariateDistributionType.Multinomial; } + } + + /// + public override string DisplayName + { + get { return "Multinomial"; } + } + + /// + public override string ShortDisplayName + { + get { return "Mult"; } + } + + /// + public override bool ParametersValid + { + get + { + if (_p == null || _p.Length < 2 || _n < 1) return false; + double sum = 0; + for (int i = 0; i < _p.Length; i++) + { + if (double.IsNaN(_p[i]) || _p[i] < 0 || _p[i] > 1) return false; + sum += _p[i]; + } + return Math.Abs(sum - 1.0) <= 1e-10; + } + } + + /// + /// Gets the mean vector. Mean[i] = N * p[i]. + /// + public double[] Mean + { + get + { + var mean = new double[_dimension]; + for (int i = 0; i < _dimension; i++) + mean[i] = _n * _p[i]; + return mean; + } + } + + /// + /// Gets the marginal variance vector. Var[i] = N * p[i] * (1 - p[i]). + /// + public double[] Variance + { + get + { + var variance = new double[_dimension]; + for (int i = 0; i < _dimension; i++) + variance[i] = _n * _p[i] * (1.0 - _p[i]); + return variance; + } + } + + /// + /// Gets the covariance between components i and j: Cov(Xᵢ, Xⱼ) = -N * pᵢ * pⱼ. + /// + /// The first component index (0-based). + /// The second component index (0-based). + /// The covariance. Negative for i != j. + public double Covariance(int i, int j) + { + if (i < 0 || i >= _dimension || j < 0 || j >= _dimension) + throw new ArgumentOutOfRangeException("Index out of range."); + if (i == j) return _n * _p[i] * (1.0 - _p[i]); + return -_n * _p[i] * _p[j]; + } + + /// + /// Computes the probability mass function (PMF) for a given count vector x. + /// + /// The count vector. Each xᵢ must be a non-negative integer and Σxᵢ = N. + /// The probability P(X = x). Returns 0 if x is not a valid count vector. + /// + /// + /// The PMF is: P(X = x) = N! / (x₁! x₂! ... xₖ!) * p₁^x₁ * p₂^x₂ * ... * pₖ^xₖ + /// Computed in log-space for numerical stability. + /// + /// + public override double PDF(double[] x) + { + return Math.Exp(LogPMF(x)); + } + + /// + /// Computes the log of the probability mass function. + /// + /// The count vector. Each xᵢ must be a non-negative integer and Σxᵢ = N. + /// The log probability. Returns if x is not valid. + public double LogPMF(double[] x) + { + if (x == null || x.Length != _dimension) return double.MinValue; + + // Validate: all non-negative integers that sum to N + int sum = 0; + for (int i = 0; i < _dimension; i++) + { + int xi = (int)Math.Round(x[i]); + if (xi < 0 || Math.Abs(x[i] - xi) > 1e-10) return double.MinValue; + sum += xi; + } + if (sum != _n) return double.MinValue; + + // Compute in log-space: log(N!) - sum(log(xi!)) + sum(xi * log(pi)) + double logPmf = Factorial.LogFactorial(_n); + for (int i = 0; i < _dimension; i++) + { + int xi = (int)Math.Round(x[i]); + logPmf -= Factorial.LogFactorial(xi); + if (xi > 0) + { + if (_p[i] <= 0) return double.MinValue; + logPmf += xi * Math.Log(_p[i]); + } + } + return logPmf; + } + + /// + public override double LogPDF(double[] x) + { + return LogPMF(x); + } + + /// + /// The CDF of the multinomial distribution is not available in closed form. + /// + /// The vector of x values. + /// This method always throws . + /// Always thrown. + public override double CDF(double[] x) + { + throw new NotImplementedException("The CDF of the multinomial distribution does not have a closed-form expression."); + } + + /// + public override MultivariateDistribution Clone() + { + var clone = new Multinomial(); + clone._n = _n; + clone._p = (double[])_p.Clone(); + clone._dimension = _dimension; + return clone; + } + + /// + /// Generates random samples from the multinomial distribution using sequential binomial sampling. + /// + /// The number of samples to generate. + /// Optional seed for reproducibility. Use -1 for a random seed. + /// A 2D array of shape [sampleSize, Dimension]. Each row sums to N. + /// + /// + /// For each sample, draws are made sequentially: for category i, draw from + /// Binomial(n_remaining, p_i / p_remaining). This is exact and efficient. + /// + /// + public double[,] GenerateRandomValues(int sampleSize, int seed = -1) + { + var rng = seed > 0 ? new MersenneTwister(seed) : new MersenneTwister(); + var sample = new double[sampleSize, _dimension]; + + for (int s = 0; s < sampleSize; s++) + { + int nRemaining = _n; + double pRemaining = 1.0; + + for (int i = 0; i < _dimension - 1; i++) + { + if (nRemaining == 0 || pRemaining <= 0) + { + sample[s, i] = 0; + continue; + } + + double conditionalP = _p[i] / pRemaining; + if (conditionalP >= 1.0) + { + sample[s, i] = nRemaining; + nRemaining = 0; + } + else + { + // Draw from Binomial(nRemaining, conditionalP) + int count = BinomialSample(nRemaining, conditionalP, rng); + sample[s, i] = count; + nRemaining -= count; + } + pRemaining -= _p[i]; + } + + // Last category gets the remainder + sample[s, _dimension - 1] = nRemaining; + } + + return sample; + } + + /// + /// Samples a single index from a discrete set of categories weighted by the given probabilities. + /// + /// Unnormalized weights. Must be non-negative with at least one positive value. + /// The random number generator. + /// The 0-based index of the selected category. + /// + /// + /// This is a weighted categorical sampling method used by the NUTS algorithm + /// to select a trajectory state weighted by exp(H). + /// + /// + /// Thrown if all weights are zero or the array is empty. + public static int Sample(double[] weights, Random rng) + { + if (weights == null || weights.Length == 0) + throw new ArgumentException("Weights array must be non-empty.", nameof(weights)); + + double totalWeight = 0; + for (int i = 0; i < weights.Length; i++) + { + if (weights[i] < 0) throw new ArgumentException("Weights must be non-negative.", nameof(weights)); + totalWeight += weights[i]; + } + if (totalWeight <= 0) + throw new ArgumentException("At least one weight must be positive.", nameof(weights)); + + double u = rng.NextDouble() * totalWeight; + double cumulative = 0; + for (int i = 0; i < weights.Length; i++) + { + cumulative += weights[i]; + if (u <= cumulative) + return i; + } + + // Should not reach here, but return last index as a safeguard + return weights.Length - 1; + } + + /// + /// Samples from a Binomial(n, p) distribution using the inverse CDF method for small n + /// and the BTPE algorithm for large n. + /// + private static int BinomialSample(int n, double p, MersenneTwister rng) + { + if (n <= 0 || p <= 0) return 0; + if (p >= 1.0) return n; + + // For small n, use direct simulation + if (n < 25) + { + int count = 0; + for (int i = 0; i < n; i++) + { + if (rng.NextDouble() < p) + count++; + } + return count; + } + + // For larger n, use the inverse CDF via the existing Binomial distribution + var binom = new Binomial(p, n); + return (int)Math.Round(binom.InverseCDF(rng.NextDouble())); + } + + } +} diff --git a/Numerics/Distributions/Multivariate/MultivariateNormal.cs b/Numerics/Distributions/Multivariate/MultivariateNormal.cs index dbdcad75..27a60e49 100644 --- a/Numerics/Distributions/Multivariate/MultivariateNormal.cs +++ b/Numerics/Distributions/Multivariate/MultivariateNormal.cs @@ -89,24 +89,24 @@ public MultivariateNormal(double[] mean, double[,] covariance) private bool _parametersValid = true; private int _dimension = 0; - private double[] _mean = Array.Empty(); - private Matrix _covariance = Matrix.Identity(0); - - private CholeskyDecomposition? _cholesky; + private double[] _mean = null!; + private Matrix _covariance = null!; + + private CholeskyDecomposition _cholesky = null!; private double _lnconstant; - private double[] _variance = Array.Empty(); - private double[] _standardDeviation = Array.Empty(); + private double[]? _variance; + private double[]? _standardDeviation; // variables required for the multivariate CDF - private Matrix _correlation = Matrix.Identity(0); - private double[] _correl = Array.Empty(); + private Matrix _correlation = null!; + private double[] _correl = null!; private Random _MVNUNI = new MersenneTwister(); private int _maxEvaluations = 100000; private double _absoluteError = 1E-4; private double _relativeError = 1E-4; - private double[] _lower = Array.Empty(); - private double[] _upper = Array.Empty(); - private int[] _infin = Array.Empty(); + private double[] _lower = null!; + private double[] _upper = null!; + private int[] _infin = null!; private bool _correlationMatrixCreated = false; private bool _covSRTed = false; @@ -250,9 +250,7 @@ public double[] StandardDeviation /// /// Determines if the covariance matrix is positive definite. /// - /// var chol = _cholesky ?? throw new InvalidOperationException("Parameters not set."); - - public bool IsPositiveDefinite => _cholesky != null && _cholesky.IsPositiveDefinite; + public bool IsPositiveDefinite => _cholesky.IsPositiveDefinite; /// /// Set the distribution parameters. @@ -267,16 +265,6 @@ public void SetParameters(double[] mean, double[,] covariance) _dimension = mean.Length; _mean = mean; _covariance = new Matrix(covariance); - - _variance = new double[_dimension]; - _standardDeviation = new double[_dimension]; - for (int i = 0; i < _dimension; i++) - { - // assuming Matrix supports indexer [row,col] - _variance[i] = _covariance[i, i]; - _standardDeviation[i] = Math.Sqrt(_variance[i]); - } - _cholesky = new CholeskyDecomposition(_covariance); double lndet = _cholesky.LogDeterminant(); _lnconstant = -(Math.Log(2d * Math.PI) * _mean.Length + lndet) * 0.5d; @@ -331,7 +319,7 @@ private void CreateCorrelationMatrix() /// The mean vector μ (mu) for the distribution. /// The covariance matrix Σ (sigma) for the distribution. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double[] mean, double[,] covariance, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double[] mean, double[,] covariance, bool throwException) { if (mean == null) { @@ -361,7 +349,7 @@ public ArgumentOutOfRangeException ValidateParameters(double[] mean, double[,] c var ex = new ArgumentOutOfRangeException(nameof(Covariance), "Covariance matrix is not positive-definite."); if (throwException) throw ex; else return ex; } - return null!; + return null; } /// @@ -396,8 +384,6 @@ public double Mahalanobis(double[] x) var z = new double[_mean.Length]; for (int i = 0; i < x.Length; i++) z[i] = x[i] - _mean[i]; - if(_cholesky == null) - throw new InvalidOperationException("Parameters not set."); var a = _cholesky.Solve(new Vector(z)); double b = 0d; for (int i = 0; i < z.Length; i++) @@ -489,9 +475,7 @@ public double[] InverseCDF(double[] probabilities) var z = new double[Dimension]; for (int j = 0; j < Dimension; j++) z[j] = Normal.StandardZ(probabilities[j]); - - if (_cholesky == null) - throw new InvalidOperationException("Parameters not set."); + // x = A*z + mu var Az = _cholesky.L * z; for (int j = 0; j < Dimension; j++) sample[j] = Az[j] + _mean[j]; @@ -554,8 +538,6 @@ public static MultivariateNormal Bivariate(double mu1, double mu2, double sigma1 for (int j = 0; j < Dimension; j++) z[j] = Normal.StandardZ(rnd.NextDouble()); // x = A*z + mu - if (_cholesky == null) - throw new InvalidOperationException("Parameters not set."); var Az = _cholesky.L * z; for (int j = 0; j < Dimension; j++) sample[i, j] = Az[j] + _mean[j]; @@ -584,8 +566,6 @@ public static MultivariateNormal Bivariate(double mu1, double mu2, double sigma1 for (int j = 0; j < Dimension; j++) z[j] = Normal.StandardZ(r[i, j]); // x = A*z + mu - if(_cholesky == null) - throw new InvalidOperationException("Parameters not set."); var Az = _cholesky.L * z; for (int j = 0; j < Dimension; j++) sample[i, j] = Az[j] + _mean[j]; @@ -621,8 +601,6 @@ public static MultivariateNormal Bivariate(double mu1, double mu2, double sigma1 } } // x = A*z + mu - if(_cholesky == null) - throw new InvalidOperationException("Parameters not set."); var Az = _cholesky.L * z; for (int j = 0; j < Dimension; j++) sample[i, j] = Az[j] + _mean[j]; @@ -636,6 +614,13 @@ public static MultivariateNormal Bivariate(double mu1, double mu2, double sigma1 //****************************************************************************80 + /// + /// Computes the bivariate normal CDF. + /// + /// Upper limit for variable X. + /// Upper limit for variable Y. + /// The correlation coefficient. + /// The bivariate normal CDF value. public static double bivnor(double ah, double ak, double r) //****************************************************************************80 @@ -1892,6 +1877,11 @@ private double BVNMVN(double[] LOWER, double[] UPPER, int[] INFIN, double CORREL return result; } + /// + /// Computes the standard normal CDF. Accurate to 1E-15. + /// + /// The Z-score. + /// The standard normal CDF value. public static double MVNPHI(double Z) { // diff --git a/Numerics/Distributions/Multivariate/MultivariateStudentT.cs b/Numerics/Distributions/Multivariate/MultivariateStudentT.cs new file mode 100644 index 00000000..5ca6b305 --- /dev/null +++ b/Numerics/Distributions/Multivariate/MultivariateStudentT.cs @@ -0,0 +1,738 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using Numerics.Mathematics.LinearAlgebra; +using Numerics.Mathematics.SpecialFunctions; +using Numerics.Sampling; + +namespace Numerics.Distributions +{ + + /// + /// The Multivariate Student's t-distribution. + /// + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// Description: + /// + /// + /// The multivariate Student's t-distribution is a generalization of the univariate Student's t-distribution + /// to multiple dimensions. It arises naturally as the marginal distribution of a multivariate normal + /// distribution with an unknown variance scaled by an inverse chi-squared random variable. It is + /// parameterized by a degrees of freedom ν, a location vector μ, and a positive-definite scale matrix Σ. + /// The scale matrix Σ is not the covariance matrix; the covariance is ν/(ν−2)·Σ for ν > 2. + /// + /// + /// The distribution has heavier tails than the multivariate normal, controlled by the degrees of freedom ν. + /// As ν → ∞, the distribution converges to the multivariate normal. For small ν, the tails are substantially + /// heavier, making it useful for robust statistical inference and confidence interval construction + /// where normal approximations underestimate tail probabilities. + /// + /// + /// References: + /// + /// + /// + /// + /// Kotz, S. and Nadarajah, S. (2004). "Multivariate t Distributions and Their Applications." + /// Cambridge University Press. + /// + /// + /// Wikipedia contributors, "Multivariate t-distribution." Wikipedia, The Free Encyclopedia. + /// Available at: + /// + /// + /// + /// + [Serializable] + public class MultivariateStudentT : MultivariateDistribution + { + + /// + /// Private parameterless constructor for use by Clone(). + /// + private MultivariateStudentT() { } + + /// + /// Constructs a standard multivariate Student's t-distribution with zero location vector, + /// identity scale matrix, and the specified degrees of freedom. + /// + /// The number of dimensions in the distribution. + /// The degrees of freedom ν (nu). Must be greater than zero. + /// Thrown when is not positive, + /// or is not positive. + public MultivariateStudentT(int dimension, double degreesOfFreedom) + { + var location = new double[dimension]; + SetParameters(degreesOfFreedom, location, Matrix.Identity(dimension).ToArray()); + } + + /// + /// Constructs a multivariate Student's t-distribution with the specified location vector, + /// identity scale matrix, and degrees of freedom. + /// + /// The degrees of freedom ν (nu). Must be greater than zero. + /// The location vector μ (mu) for the distribution. + /// Thrown when is not positive, + /// or is null. + public MultivariateStudentT(double degreesOfFreedom, double[] location) + { + SetParameters(degreesOfFreedom, location, Matrix.Identity(location.Length).ToArray()); + } + + /// + /// Constructs a multivariate Student's t-distribution with the specified degrees of freedom, + /// location vector, and scale matrix. + /// + /// The degrees of freedom ν (nu). Must be greater than zero. + /// The location vector μ (mu) for the distribution. + /// The scale matrix Σ (sigma) for the distribution. Must be positive definite. + /// Note: this is the scale matrix, not the covariance matrix. The covariance is ν/(ν−2)·Σ for ν > 2. + /// Thrown when parameters are invalid. + public MultivariateStudentT(double degreesOfFreedom, double[] location, double[,] scaleMatrix) + { + SetParameters(degreesOfFreedom, location, scaleMatrix); + } + + private bool _parametersValid = true; + private int _dimension; + private double _degreesOfFreedom; + private double[] _location = null!; + private Matrix _scaleMatrix = null!; + + private CholeskyDecomposition _cholesky = null!; + private double _lnconstant; + private double[]? _variance; + private double[]? _standardDeviation; + + /// + /// Gets the number of variables for the distribution. + /// + public override int Dimension + { + get { return _dimension; } + } + + /// + /// Returns the multivariate distribution type. + /// + public override MultivariateDistributionType Type + { + get { return MultivariateDistributionType.MultivariateStudentT; } + } + + /// + /// Returns the display name of the distribution type as a string. + /// + public override string DisplayName + { + get { return "Multivariate Student's t"; } + } + + /// + /// Returns the short display name of the distribution as a string. + /// + public override string ShortDisplayName + { + get { return "Multi-T"; } + } + + /// + /// Determines whether the parameters are valid or not. + /// + public override bool ParametersValid + { + get { return _parametersValid; } + } + + /// + /// Gets the degrees of freedom ν (nu) of the distribution. + /// + public double DegreesOfFreedom + { + get { return _degreesOfFreedom; } + } + + /// + /// Gets the location vector μ (mu) of the distribution. + /// + public double[] Location + { + get { return _location; } + } + + /// + /// Gets the mean vector of the distribution. Equal to the location vector μ when ν > 1. + /// + /// Thrown when ν ≤ 1, as the mean is undefined. + public double[] Mean + { + get + { + if (_degreesOfFreedom <= 1.0) + throw new InvalidOperationException("The mean is undefined for degrees of freedom ν ≤ 1."); + return _location; + } + } + + /// + /// Gets the median vector of the distribution. Equal to the location vector μ. + /// + public double[] Median + { + get { return _location; } + } + + /// + /// Gets the mode vector of the distribution. Equal to the location vector μ. + /// + public double[] Mode + { + get { return _location; } + } + + /// + /// Gets the marginal variance vector of the distribution: ν/(ν−2) · diag(Σ). + /// + /// Thrown when ν ≤ 2, as the variance is undefined. + public double[] Variance + { + get + { + if (_degreesOfFreedom <= 2.0) + throw new InvalidOperationException("The variance is undefined for degrees of freedom ν ≤ 2."); + if (_variance == null) + { + double scale = _degreesOfFreedom / (_degreesOfFreedom - 2.0); + var diag = _scaleMatrix.Diagonal(); + _variance = new double[Dimension]; + for (int i = 0; i < Dimension; i++) + _variance[i] = scale * diag[i]; + } + return _variance; + } + } + + /// + /// Gets the marginal standard deviation vector of the distribution. + /// + /// Thrown when ν ≤ 2, as the variance is undefined. + public double[] StandardDeviation + { + get + { + if (_standardDeviation == null) + { + _standardDeviation = new double[Dimension]; + var v = Variance; + for (int i = 0; i < Dimension; i++) + _standardDeviation[i] = Math.Sqrt(v[i]); + } + return _standardDeviation; + } + } + + /// + /// Gets the scale matrix Σ (sigma) of the distribution. + /// + /// + /// The scale matrix is not the covariance matrix. The covariance matrix is ν/(ν−2)·Σ for ν > 2. + /// + public double[,] ScaleMatrix + { + get { return _scaleMatrix.ToArray(); } + } + + /// + /// Gets the covariance matrix of the distribution: ν/(ν−2) · Σ. + /// + /// Thrown when ν ≤ 2, as the covariance is undefined. + public double[,] Covariance + { + get + { + if (_degreesOfFreedom <= 2.0) + throw new InvalidOperationException("The covariance is undefined for degrees of freedom ν ≤ 2."); + double scale = _degreesOfFreedom / (_degreesOfFreedom - 2.0); + var cov = _scaleMatrix.Clone(); + for (int i = 0; i < Dimension; i++) + for (int j = 0; j < Dimension; j++) + cov[i, j] *= scale; + return cov.ToArray(); + } + } + + /// + /// Determines if the scale matrix is positive definite. + /// + public bool IsPositiveDefinite => _cholesky.IsPositiveDefinite; + + /// + /// Set the distribution parameters. + /// + /// The degrees of freedom ν (nu). Must be greater than zero. + /// The location vector μ (mu) for the distribution. + /// The scale matrix Σ (sigma) for the distribution. Must be positive definite. + /// Thrown when parameters are invalid. + public void SetParameters(double degreesOfFreedom, double[] location, double[,] scaleMatrix) + { + // Validate parameters + ValidateParameters(degreesOfFreedom, location, scaleMatrix, true); + + _degreesOfFreedom = degreesOfFreedom; + _dimension = location.Length; + _location = location; + _scaleMatrix = new Matrix(scaleMatrix); + _cholesky = new CholeskyDecomposition(_scaleMatrix); + + // Precompute the log of the normalizing constant for the PDF: + // ln C = LogGamma((ν+p)/2) - LogGamma(ν/2) - (p/2)*ln(νπ) - (1/2)*ln|Σ| + double lndet = _cholesky.LogDeterminant(); + _lnconstant = Gamma.LogGamma((_degreesOfFreedom + _dimension) / 2.0) + - Gamma.LogGamma(_degreesOfFreedom / 2.0) + - (_dimension / 2.0) * Math.Log(_degreesOfFreedom * Math.PI) + - 0.5 * lndet; + + // Reset cached derived properties + _variance = null; + _standardDeviation = null; + } + + /// + /// Validate the distribution parameters. + /// + /// The degrees of freedom ν (nu). + /// The location vector μ (mu). + /// The scale matrix Σ (sigma). + /// Determines whether to throw an exception or return it. + /// + /// An if validation fails; otherwise null. + /// + public ArgumentOutOfRangeException? ValidateParameters(double degreesOfFreedom, double[] location, double[,] scaleMatrix, bool throwException) + { + if (degreesOfFreedom <= 0) + { + var ex = new ArgumentOutOfRangeException(nameof(degreesOfFreedom), "Degrees of freedom must be greater than zero."); + if (throwException) throw ex; else return ex; + } + if (location == null) + { + var ex = new ArgumentOutOfRangeException(nameof(location), "Location vector must not be null."); + if (throwException) throw ex; else return ex; + } + if (scaleMatrix == null) + { + var ex = new ArgumentOutOfRangeException(nameof(scaleMatrix), "Scale matrix must not be null."); + if (throwException) throw ex; else return ex; + } + var m = new Matrix(scaleMatrix); + if (!m.IsSquare) + { + var ex = new ArgumentOutOfRangeException(nameof(scaleMatrix), "Scale matrix must be square."); + if (throwException) throw ex; else return ex; + } + if (m.NumberOfRows != location.Length) + { + var ex = new ArgumentOutOfRangeException(nameof(scaleMatrix), "Location vector length must match scale matrix dimension."); + if (throwException) throw ex; else return ex; + } + try + { + var chol = new CholeskyDecomposition(m); + if (!chol.IsPositiveDefinite) + { + var ex = new ArgumentOutOfRangeException(nameof(scaleMatrix), "Scale matrix is not positive-definite."); + if (throwException) throw ex; else return ex; + } + } + catch (ArgumentOutOfRangeException) { throw; } + catch (Exception) + { + var ex = new ArgumentOutOfRangeException(nameof(scaleMatrix), "Scale matrix is not positive-definite."); + if (throwException) throw ex; else return ex; + } + return null; + } + + /// + /// The Probability Density Function (PDF) of the distribution evaluated at a point x. + /// + /// A point in the distribution space. + /// The probability density at point x. + /// + /// + /// The PDF of the multivariate Student's t-distribution is: + /// + /// + /// f(x) = C · [1 + (x−μ)'Σ⁻¹(x−μ)/ν]^(−(ν+p)/2) + /// + /// + /// where C = Γ((ν+p)/2) / [Γ(ν/2) · (νπ)^(p/2) · |Σ|^(1/2)]. + /// + /// + public override double PDF(double[] x) + { + return Math.Exp(LogPDF(x)); + } + + /// + /// Returns the natural log of the PDF evaluated at a point x. + /// + /// A point in the distribution space. + /// The natural logarithm of the probability density at point x. + /// + /// + /// Uses the numerically stable formulation: + /// + /// + /// log f(x) = ln C − ((ν+p)/2) · ln(1 + Q/ν) + /// + /// + /// where Q = (x−μ)'Σ⁻¹(x−μ) is the squared Mahalanobis distance, and ln C is the precomputed + /// log normalizing constant. + /// + /// + public override double LogPDF(double[] x) + { + double Q = Mahalanobis(x); + double f = _lnconstant - ((_degreesOfFreedom + _dimension) / 2.0) * Math.Log(1.0 + Q / _degreesOfFreedom); + if (double.IsNaN(f) || double.IsInfinity(f)) return double.MinValue; + return f; + } + + /// + /// Gets the squared Mahalanobis distance between a sample point and this distribution: + /// (x−μ)'Σ⁻¹(x−μ). + /// + /// A point in the distribution space. + /// The squared Mahalanobis distance. + /// Thrown when the vector dimension does not match + /// the distribution dimension. + public double Mahalanobis(double[] x) + { + if (x.Length != Dimension) + throw new ArgumentOutOfRangeException(nameof(x), "The vector must be the same dimension as the distribution."); + var z = new double[_location.Length]; + for (int i = 0; i < x.Length; i++) + z[i] = x[i] - _location[i]; + var a = _cholesky.Solve(new Vector(z)); + double b = 0.0; + for (int i = 0; i < z.Length; i++) + b += a[i] * z[i]; + return b; + } + + /// + /// The Cumulative Distribution Function (CDF) for the distribution evaluated at a point x. + /// + /// A point in the distribution space. + /// The cumulative probability P(X ≤ x). + /// + /// + /// For dimension 1, this delegates to the univariate Student's t CDF. + /// + /// + /// For dimensions ≥ 2, the CDF is computed via stratified numerical integration over the + /// mixing representation: P(X ≤ x) = E_W[ Φ_p((x−μ)·√(W/ν); 0, Σ) ] where W ~ χ²(ν), + /// and Φ_p is the multivariate normal CDF. The expectation is computed by evaluating the + /// MVN CDF at K=200 stratified quantiles of the χ²(ν) distribution and averaging. + /// + /// + /// Reference: Genz, A. and Bretz, F. (2009). "Computation of Multivariate Normal and t Probabilities." + /// Lecture Notes in Statistics, Vol. 195. Springer. + /// + /// + public override double CDF(double[] x) + { + if (x.Length != Dimension) + throw new ArgumentOutOfRangeException(nameof(x), "The vector must be the same dimension as the distribution."); + + if (Dimension == 1) + { + // Delegate to univariate Student's t CDF + double sigma = Math.Sqrt(_scaleMatrix[0, 0]); + var univT = new StudentT(_location[0], sigma, _degreesOfFreedom); + return univT.CDF(x[0]); + } + + // For D >= 2, use stratified quantile integration over the χ²(ν) mixing variable. + // The multivariate-t CDF can be written as: + // P(X ≤ x) = E_W[ Φ_MVN((x−μ)·√(W/ν); 0, Σ) ] where W ~ χ²(ν) + // + // We evaluate this by computing the MVN CDF at K equally-spaced quantiles + // of χ²(ν) and averaging. This is deterministic and works for any ν. + + const int K = 200; + var gamma = new GammaDistribution(2.0, _degreesOfFreedom / 2.0); + + // Centered point for MVN CDF evaluation + var zVec = new double[Dimension]; + for (int i = 0; i < Dimension; i++) + zVec[i] = x[i] - _location[i]; + + // Create MVN with zero mean and the scale matrix Σ for CDF evaluation + var mvn = new MultivariateNormal(new double[Dimension], _scaleMatrix.ToArray()); + + double sum = 0.0; + for (int k = 0; k < K; k++) + { + // Use midpoint of each stratum for stratified integration + double p = (k + 0.5) / K; + double w = gamma.InverseCDF(p); + double scaleFactor = Math.Sqrt(w / _degreesOfFreedom); + + // Scale the centered vector: (x-μ) · √(W/ν) + var scaledZ = new double[Dimension]; + for (int i = 0; i < Dimension; i++) + scaledZ[i] = zVec[i] * scaleFactor; + + sum += mvn.CDF(scaledZ); + } + + double result = sum / K; + + // Clamp to [0, 1] + return Math.Max(0.0, Math.Min(1.0, result)); + } + + /// + /// Generate a matrix of random values from the multivariate Student's t-distribution. + /// + /// Size of random sample to generate. + /// Optional seed for the random number generator. Use -1 for a random seed. + /// + /// A 2D array of random values. The number of rows equals the sample size. + /// The number of columns equals the dimension of the distribution. + /// + /// + /// + /// Sampling uses the representation: X = μ + L·z / √(W/ν), + /// where L is the Cholesky factor of Σ, z ~ N(0, I_p), and W ~ χ²(ν). + /// The χ² variate is generated via the equivalent Gamma(ν/2, 2) distribution + /// to support non-integer degrees of freedom. + /// + /// + public double[,] GenerateRandomValues(int sampleSize, int seed = -1) + { + var rnd = seed > 0 ? new MersenneTwister(seed) : new MersenneTwister(); + var sample = new double[sampleSize, Dimension]; + + // Use Gamma(ν/2, 2) to generate χ²(ν) variates, supporting non-integer ν + var gamma = new GammaDistribution(2.0, _degreesOfFreedom / 2.0); + + for (int i = 0; i < sampleSize; i++) + { + // z ~ N(0, I_p) + var z = new double[Dimension]; + for (int j = 0; j < Dimension; j++) + z[j] = Normal.StandardZ(rnd.NextDouble()); + + // W ~ χ²(ν) via Gamma(ν/2, 2) + double w = gamma.InverseCDF(rnd.NextDouble()); + + // scale = √(ν / W) + double scale = Math.Sqrt(_degreesOfFreedom / w); + + // x = μ + L·z · scale + var Lz = _cholesky.L * z; + for (int j = 0; j < Dimension; j++) + sample[i, j] = _location[j] + Lz[j] * scale; + } + return sample; + } + + /// + /// Generate random values using Latin Hypercube Sampling (LHS) for improved space-filling properties. + /// + /// Size of random sample to generate. + /// Seed for the random number generator. + /// + /// A 2D array of random values with improved coverage of the distribution space. + /// + /// + /// + /// Latin Hypercube Sampling ensures that each marginal dimension is evenly stratified. + /// The LHS uniforms are used for the normal component z ~ N(0, I_p), while a separate + /// independent random stream generates the χ²(ν) mixing variates. This preserves the + /// space-filling properties of LHS in the correlated normal dimensions while maintaining + /// correct marginal t-distribution behavior through the independent χ² scaling. + /// + /// + public double[,] LatinHypercubeRandomValues(int sampleSize, int seed) + { + var r = LatinHypercube.Random(sampleSize, Dimension, seed); + var sample = new double[sampleSize, Dimension]; + + // Separate PRNG stream for χ² variates (independent of the LHS grid) + var rndChi = new MersenneTwister(seed + 1); + var gamma = new GammaDistribution(2.0, _degreesOfFreedom / 2.0); + + for (int i = 0; i < sampleSize; i++) + { + // z ~ N(0, I_p) via LHS uniforms + var z = new double[Dimension]; + for (int j = 0; j < Dimension; j++) + z[j] = Normal.StandardZ(r[i, j]); + + // W ~ χ²(ν) via independent stream + double w = gamma.InverseCDF(rndChi.NextDouble()); + double scale = Math.Sqrt(_degreesOfFreedom / w); + + // x = μ + L·z · scale + var Lz = _cholesky.L * z; + for (int j = 0; j < Dimension; j++) + sample[i, j] = _location[j] + Lz[j] * scale; + } + return sample; + } + + /// + /// Returns a 2D array of stratified random variates. The first dimension is stratified, + /// and the remaining dimensions are sampled randomly. + /// + /// A list of stratification bins defining the stratified dimension. + /// Seed for the random number generator. + /// + /// A 2D array of stratified random values. + /// + /// + /// + /// The first marginal dimension uses stratified uniform quantiles (bin midpoints), + /// while the remaining dimensions use independent random uniforms. The χ²(ν) mixing + /// variate is drawn from a separate random stream. + /// + /// + public double[,] StratifiedRandomValues(List stratificationBins, int seed) + { + int samplesize = stratificationBins.Count; + var rnd = new MersenneTwister(seed); + var rndChi = new MersenneTwister(seed + 1); + var gamma = new GammaDistribution(2.0, _degreesOfFreedom / 2.0); + var sample = new double[samplesize, Dimension]; + + for (int i = 0; i < samplesize; i++) + { + // z ~ N(0, I_p) with first dimension stratified + var z = new double[Dimension]; + for (int j = 0; j < Dimension; j++) + { + if (j == 0) + z[j] = Normal.StandardZ(stratificationBins[i].Midpoint); + else + z[j] = Normal.StandardZ(rnd.NextDouble()); + } + + // W ~ χ²(ν) via independent stream + double w = gamma.InverseCDF(rndChi.NextDouble()); + double scale = Math.Sqrt(_degreesOfFreedom / w); + + // x = μ + L·z · scale + var Lz = _cholesky.L * z; + for (int j = 0; j < Dimension; j++) + sample[i, j] = _location[j] + Lz[j] * scale; + } + return sample; + } + + /// + /// The inverse cumulative distribution function (InverseCDF). + /// + /// + /// Array of p+1 uniform probabilities in (0, 1). The first p entries correspond to the + /// correlated normal dimensions, and the last entry is the probability for the χ²(ν) + /// mixing variable that controls the tail thickness. + /// + /// A p-dimensional sample point from the distribution. + /// Thrown when the length of + /// is not equal to Dimension + 1. + /// + /// + /// Uses the representation X = μ + L·z · √(ν/W), where z_j = Φ⁻¹(p_j) for j = 1..p + /// and W = F_{χ²(ν)}⁻¹(p_{p+1}). L is the Cholesky factor of the scale matrix Σ. + /// + /// + /// The extra probability (compared to ) controls + /// the χ² mixing variate. Values near 0 produce extreme tail samples (large scaling), + /// while values near 1 produce samples close to the multivariate normal. + /// + /// + public double[] InverseCDF(double[] probabilities) + { + if (probabilities.Length != Dimension + 1) + throw new ArgumentOutOfRangeException(nameof(probabilities), + $"The probabilities array must have length {Dimension + 1} (Dimension + 1 for the χ² mixing variable)."); + + // Convert first p probabilities to standard normal variates + var z = new double[Dimension]; + for (int j = 0; j < Dimension; j++) + z[j] = Normal.StandardZ(probabilities[j]); + + // Convert last probability to χ²(ν) variate via Gamma(ν/2, 2) + var gamma = new GammaDistribution(2.0, _degreesOfFreedom / 2.0); + double w = gamma.InverseCDF(probabilities[Dimension]); + double scale = Math.Sqrt(_degreesOfFreedom / w); + + // x = μ + L·z · scale + var Lz = _cholesky.L * z; + var sample = new double[Dimension]; + for (int j = 0; j < Dimension; j++) + sample[j] = _location[j] + Lz[j] * scale; + + return sample; + } + + /// + /// Creates a deep copy of this distribution. + /// + /// A new instance with identical parameters. + public override MultivariateDistribution Clone() + { + var clone = new MultivariateStudentT() + { + _parametersValid = this._parametersValid, + _dimension = this._dimension, + _degreesOfFreedom = this._degreesOfFreedom, + _location = this._location.ToArray(), + _scaleMatrix = this._scaleMatrix.Clone(), + _cholesky = new CholeskyDecomposition(this._scaleMatrix.Clone()), + _lnconstant = this._lnconstant, + }; + return clone; + } + + } +} \ No newline at end of file diff --git a/Numerics/Distributions/Univariate/Base/IUnivariateDistribution.cs b/Numerics/Distributions/Univariate/Base/IUnivariateDistribution.cs index 3c7b7f83..d84e59fb 100644 --- a/Numerics/Distributions/Univariate/Base/IUnivariateDistribution.cs +++ b/Numerics/Distributions/Univariate/Base/IUnivariateDistribution.cs @@ -159,7 +159,7 @@ public interface IUnivariateDistribution : IDistribution /// Array of parameters. /// Boolean indicating whether to throw the exception or not. /// Nothing if the parameters are valid and the exception if invalid parameters were found. - ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException); + ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException); /// /// The Probability Density Function (PDF) of the distribution evaluated at a point X. diff --git a/Numerics/Distributions/Univariate/Base/UnivariateDistributionBase.cs b/Numerics/Distributions/Univariate/Base/UnivariateDistributionBase.cs index 08b96017..5a759c25 100644 --- a/Numerics/Distributions/Univariate/Base/UnivariateDistributionBase.cs +++ b/Numerics/Distributions/Univariate/Base/UnivariateDistributionBase.cs @@ -164,7 +164,7 @@ public double CoefficientOfVariation public abstract void SetParameters(IList parameters); /// - public abstract ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException); + public abstract ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException); /// /// The log likelihood function. diff --git a/Numerics/Distributions/Univariate/Base/UnivariateDistributionFactory.cs b/Numerics/Distributions/Univariate/Base/UnivariateDistributionFactory.cs index 84570380..c631a415 100644 --- a/Numerics/Distributions/Univariate/Base/UnivariateDistributionFactory.cs +++ b/Numerics/Distributions/Univariate/Base/UnivariateDistributionFactory.cs @@ -55,160 +55,153 @@ public sealed class UnivariateDistributionFactory /// public static UnivariateDistributionBase CreateDistribution(UnivariateDistributionType distributionType) { - var distribution = default(UnivariateDistributionBase); if (distributionType == UnivariateDistributionType.Bernoulli) { - distribution = new Bernoulli(); + return new Bernoulli(); } else if (distributionType == UnivariateDistributionType.Beta) { - distribution = new BetaDistribution(); + return new BetaDistribution(); } else if (distributionType == UnivariateDistributionType.Binomial) { - distribution = new Binomial(); + return new Binomial(); } else if (distributionType == UnivariateDistributionType.Cauchy) { - distribution = new Cauchy(); + return new Cauchy(); } else if (distributionType == UnivariateDistributionType.ChiSquared) { - distribution = new ChiSquared(); - } - else if (distributionType == UnivariateDistributionType.Deterministic) - { - distribution = new Deterministic(); + return new ChiSquared(); } else if (distributionType == UnivariateDistributionType.Exponential) { - distribution = new Exponential(); + return new Exponential(); } else if (distributionType == UnivariateDistributionType.GammaDistribution) { - distribution = new GammaDistribution(); + return new GammaDistribution(); } else if (distributionType == UnivariateDistributionType.GeneralizedBeta) { - distribution = new GeneralizedBeta(); + return new GeneralizedBeta(); } else if (distributionType == UnivariateDistributionType.GeneralizedExtremeValue) { - distribution = new GeneralizedExtremeValue(); + return new GeneralizedExtremeValue(); } else if (distributionType == UnivariateDistributionType.GeneralizedLogistic) { - distribution = new GeneralizedLogistic(); + return new GeneralizedLogistic(); } else if (distributionType == UnivariateDistributionType.GeneralizedNormal) { - distribution = new GeneralizedNormal(); + return new GeneralizedNormal(); } else if (distributionType == UnivariateDistributionType.GeneralizedPareto) { - distribution = new GeneralizedPareto(); + return new GeneralizedPareto(); } else if (distributionType == UnivariateDistributionType.Geometric) { - distribution = new Geometric(); + return new Geometric(); } else if (distributionType == UnivariateDistributionType.Gumbel) { - distribution = new Gumbel(); + return new Gumbel(); } else if (distributionType == UnivariateDistributionType.InverseChiSquared) { - distribution = new InverseChiSquared(); + return new InverseChiSquared(); } else if (distributionType == UnivariateDistributionType.InverseGamma) { - distribution = new InverseGamma(); + return new InverseGamma(); } else if (distributionType == UnivariateDistributionType.KappaFour) { - distribution = new KappaFour(); + return new KappaFour(); } else if (distributionType == UnivariateDistributionType.LnNormal) { - distribution = new LnNormal(); + return new LnNormal(); } else if (distributionType == UnivariateDistributionType.Logistic) { - distribution = new Logistic(); + return new Logistic(); } else if (distributionType == UnivariateDistributionType.LogNormal) { - distribution = new LogNormal(); + return new LogNormal(); } else if (distributionType == UnivariateDistributionType.LogPearsonTypeIII) { - distribution = new LogPearsonTypeIII(); + return new LogPearsonTypeIII(); } else if (distributionType == UnivariateDistributionType.NoncentralT) { - distribution = new NoncentralT(); + return new NoncentralT(); } else if (distributionType == UnivariateDistributionType.Normal) { - distribution = new Normal(); + return new Normal(); } else if (distributionType == UnivariateDistributionType.Pareto) { - distribution = new Pareto(); + return new Pareto(); } else if (distributionType == UnivariateDistributionType.PearsonTypeIII) { - distribution = new PearsonTypeIII(); + return new PearsonTypeIII(); } else if (distributionType == UnivariateDistributionType.Pert) { - distribution = new Pert(); + return new Pert(); } else if (distributionType == UnivariateDistributionType.PertPercentile) { - distribution = new PertPercentile(); + return new PertPercentile(); } else if (distributionType == UnivariateDistributionType.PertPercentileZ) { - distribution = new PertPercentileZ(); + return new PertPercentileZ(); } else if (distributionType == UnivariateDistributionType.Poisson) { - distribution = new Poisson(); + return new Poisson(); } else if (distributionType == UnivariateDistributionType.Rayleigh) { - distribution = new Rayleigh(); + return new Rayleigh(); } else if (distributionType == UnivariateDistributionType.StudentT) { - distribution = new StudentT(); + return new StudentT(); } else if (distributionType == UnivariateDistributionType.Triangular) { - distribution = new Triangular(); + return new Triangular(); } else if (distributionType == UnivariateDistributionType.TruncatedNormal) { - distribution = new TruncatedNormal(); + return new TruncatedNormal(); } else if (distributionType == UnivariateDistributionType.Uniform) { - distribution = new Uniform(); + return new Uniform(); } else if (distributionType == UnivariateDistributionType.UniformDiscrete) { - distribution = new UniformDiscrete(); + return new UniformDiscrete(); } else if (distributionType == UnivariateDistributionType.Weibull) { - distribution = new Weibull(); - } - if (distribution is null) - { - throw new ArgumentException("Distribution is not found."); + return new Weibull(); } - return distribution; + + // Default to Deterministic for unrecognized types + return new Deterministic(); } /// @@ -218,54 +211,42 @@ public static UnivariateDistributionBase CreateDistribution(UnivariateDistributi /// /// A univariate distribution. /// - public static UnivariateDistributionBase? CreateDistribution(XElement xElement) + public static UnivariateDistributionBase CreateDistribution(XElement xElement) { - UnivariateDistributionType type; - UnivariateDistributionBase? dist = null; - var xAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); - if (xAttr != null) + UnivariateDistributionType type = UnivariateDistributionType.Deterministic; + var typeAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); + if (typeAttr != null) { - Enum.TryParse(xAttr.Value, out type); + Enum.TryParse(typeAttr.Value, out type); if (type == UnivariateDistributionType.Mixture) { - dist = Mixture.FromXElement(xElement); - return dist; + return Mixture.FromXElement(xElement)!; } else if (type == UnivariateDistributionType.CompetingRisks) { - dist = CompetingRisks.FromXElement(xElement); - return dist; + return CompetingRisks.FromXElement(xElement)!; } else if (type == UnivariateDistributionType.PertPercentile) { - dist = PertPercentile.FromXElement(xElement); - return dist; + return PertPercentile.FromXElement(xElement)!; } else if (type == UnivariateDistributionType.PertPercentileZ) { - dist = PertPercentileZ.FromXElement(xElement); - return dist; + return PertPercentileZ.FromXElement(xElement)!; } - else - { - dist = CreateDistribution(type); - } - - } - if (dist is null) - { - throw new ArgumentException("Distribution is not found."); } + + var dist = CreateDistribution(type); var names = dist.GetParameterPropertyNames; var parms = dist.GetParameters; var vals = new double[dist.NumberOfParameters]; for (int i = 0; i < dist.NumberOfParameters; i++) { - var xAttrParm = xElement.Attribute(names[i]); - if (xAttrParm != null) + var paramAttr = xElement.Attribute(names[i]); + if (paramAttr != null) { - double.TryParse(xAttrParm.Value, System.Globalization.NumberStyles.Any, System.Globalization.CultureInfo.InvariantCulture, out vals[i]); + double.TryParse(paramAttr.Value, System.Globalization.NumberStyles.Any, System.Globalization.CultureInfo.InvariantCulture, out vals[i]); } } dist.SetParameters(vals); diff --git a/Numerics/Distributions/Univariate/Base/UnivariateDistributionType.cs b/Numerics/Distributions/Univariate/Base/UnivariateDistributionType.cs index bd11640d..1d7f11ae 100644 --- a/Numerics/Distributions/Univariate/Base/UnivariateDistributionType.cs +++ b/Numerics/Distributions/Univariate/Base/UnivariateDistributionType.cs @@ -210,6 +210,10 @@ public enum UnivariateDistributionType /// UserDefined, /// + /// Von Mises (circular normal) distribution. + /// + VonMises, + /// /// Weibull distribution. /// Weibull diff --git a/Numerics/Distributions/Univariate/Bernoulli.cs b/Numerics/Distributions/Univariate/Bernoulli.cs index 72d9268e..04d47e6c 100644 --- a/Numerics/Distributions/Univariate/Bernoulli.cs +++ b/Numerics/Distributions/Univariate/Bernoulli.cs @@ -176,13 +176,21 @@ public override double StandardDeviation /// public override double Skewness { - get { return (Complement - Probability) / Math.Sqrt(Probability * Complement); } + get + { + if (Probability == 0d || Probability == 1d) return double.NaN; + return (Complement - Probability) / Math.Sqrt(Probability * Complement); + } } /// public override double Kurtosis { - get { return 3d + (1.0d - 6d * Complement * Probability) / (Probability * Complement); } + get + { + if (Probability == 0d || Probability == 1d) return double.NaN; + return 3d + (1.0d - 6d * Complement * Probability) / (Probability * Complement); + } } /// @@ -216,7 +224,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { // Validate probability if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0]) || parameters[0] < 0.0d || parameters[0] > 1.0d) diff --git a/Numerics/Distributions/Univariate/BetaDistribution.cs b/Numerics/Distributions/Univariate/BetaDistribution.cs index 6cb56ba6..96082e7b 100644 --- a/Numerics/Distributions/Univariate/BetaDistribution.cs +++ b/Numerics/Distributions/Univariate/BetaDistribution.cs @@ -239,7 +239,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0]) || parameters[0] <= 0.0d) { diff --git a/Numerics/Distributions/Univariate/Binomial.cs b/Numerics/Distributions/Univariate/Binomial.cs index 9ffc70cd..52577320 100644 --- a/Numerics/Distributions/Univariate/Binomial.cs +++ b/Numerics/Distributions/Univariate/Binomial.cs @@ -241,7 +241,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { // Validate probability if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0]) || parameters[0] < 0.0d || parameters[0] > 1.0d) @@ -253,8 +253,8 @@ public override ArgumentOutOfRangeException ValidateParameters(IList par if (double.IsNaN(parameters[1]) || double.IsInfinity(parameters[1]) || parameters[1] <= 0.0d) { if (throwException) - throw new ArgumentOutOfRangeException(nameof(ProbabilityOfSuccess), "The number of trials (n) must be positive."); - return new ArgumentOutOfRangeException(nameof(ProbabilityOfSuccess), "The number of trials (n) must be positive."); + throw new ArgumentOutOfRangeException(nameof(NumberOfTrials), "The number of trials (n) must be positive."); + return new ArgumentOutOfRangeException(nameof(NumberOfTrials), "The number of trials (n) must be positive."); } return null!; } @@ -282,7 +282,7 @@ public override double CDF(double k) k = Math.Floor(k); if (k < Minimum) return 0.0d; - if (k > Maximum) + if (k >= Maximum) return 1.0d; return Beta.Incomplete(NumberOfTrials - k, k + 1d, Complement); } @@ -299,9 +299,9 @@ public override double InverseCDF(double probability) return Maximum; // Validate parameters if (_parametersValid == false) - ValidateParameters([probability, NumberOfTrials], true); + ValidateParameters([ProbabilityOfSuccess, NumberOfTrials], true); double k = 0d; - for (int i = 0; i < NumberOfTrials; i++) + for (int i = 0; i <= NumberOfTrials; i++) { if (CDF(i) >= probability) { diff --git a/Numerics/Distributions/Univariate/Cauchy.cs b/Numerics/Distributions/Univariate/Cauchy.cs index 639c725a..47ac080a 100644 --- a/Numerics/Distributions/Univariate/Cauchy.cs +++ b/Numerics/Distributions/Univariate/Cauchy.cs @@ -215,7 +215,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0])) { diff --git a/Numerics/Distributions/Univariate/ChiSquared.cs b/Numerics/Distributions/Univariate/ChiSquared.cs index 1f6cdaa7..4eb7e708 100644 --- a/Numerics/Distributions/Univariate/ChiSquared.cs +++ b/Numerics/Distributions/Univariate/ChiSquared.cs @@ -233,7 +233,7 @@ public override void SetParameters(IList parameters) /// /// The degrees of freedom for the distribution. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(int degreesOfFreedom, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(int degreesOfFreedom, bool throwException) { if (degreesOfFreedom < 1.0d) { @@ -246,7 +246,7 @@ public ArgumentOutOfRangeException ValidateParameters(int degreesOfFreedom, bool } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters((int)parameters[0], throwException); } @@ -259,10 +259,17 @@ public override double PDF(double X) ValidateParameters(DegreesOfFreedom, true); if (X < Minimum) return 0.0d; double v = DegreesOfFreedom; - double a = Math.Pow(X, (v - 2.0d) / 2.0d); - double b = Math.Exp(-X / 2.0d); - double c = Math.Pow(2d, v / 2.0d) * Gamma.Function(v / 2.0d); - return a * b / c; + // Handle x=0 edge case: PDF(0) = 0 for v>2, 0.5 for v=2, +Inf for v<2 + if (X == 0.0d) + { + if (v > 2) return 0.0d; + if (v == 2) return 0.5d; + return double.PositiveInfinity; + } + // Compute in log-space to avoid overflow for large degrees of freedom + double logPdf = ((v - 2.0d) / 2.0d) * Math.Log(X) - X / 2.0d + - (v / 2.0d) * Math.Log(2.0d) - Gamma.LogGamma(v / 2.0d); + return Math.Exp(logPdf); } /// diff --git a/Numerics/Distributions/Univariate/CompetingRisks.cs b/Numerics/Distributions/Univariate/CompetingRisks.cs index e0ec9b98..0c640975 100644 --- a/Numerics/Distributions/Univariate/CompetingRisks.cs +++ b/Numerics/Distributions/Univariate/CompetingRisks.cs @@ -79,7 +79,7 @@ public CompetingRisks(IUnivariateDistribution[] distributions) SetParameters(distributions); } - private UnivariateDistributionBase[] _distributions = Array.Empty(); + private UnivariateDistributionBase[] _distributions = null!; private EmpiricalDistribution _empiricalCDF = null!; private bool _momentsComputed = false; private double u1, u2, u3, u4; @@ -88,6 +88,10 @@ public CompetingRisks(IUnivariateDistribution[] distributions) private bool _mvnCreated = false; private MultivariateNormal _mvn = null!; + // Minimum log value to prevent -Infinity in log-likelihood + private const double _logZero = -745.0; // ≈ log(double.MinValue that's positive) + private const double _minDensity = 1E-300; + /// /// Returns the array of univariate probability distributions. /// @@ -155,10 +159,10 @@ public override int NumberOfParameters { var parmString = new string[1, 2]; string Dstring = "{"; - for (int i = 1; i < Distributions.Count - 1; i++) + for (int i = 0; i < Distributions.Count; i++) { Dstring += Distributions[i].DisplayName; - if (i < Distributions.Count - 2) + if (i < Distributions.Count - 1) { Dstring += ","; } @@ -410,7 +414,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (Distributions.Count == 0) return new ArgumentOutOfRangeException(nameof(Distributions), "There must be at least 1 distribution"); for (int i = 0; i < Distributions.Count; i++) @@ -422,7 +426,7 @@ public override ArgumentOutOfRangeException ValidateParameters(IList par return new ArgumentOutOfRangeException(nameof(Distributions), "One of the distributions have invalid parameters."); } } - return null!; + return null; } /// @@ -490,23 +494,14 @@ public override double PDF(double x) // Only compute the exact PDF for independent random variables if (Dependency == Probability.DependencyType.Independent) { - double hf = 0d; - double sf = 1d; - for (int i = 0; i < Distributions.Count; i++) + if (MinimumOfRandomVariables) { - - if (MinimumOfRandomVariables == true) - { - hf += Distributions[i].HF(x); - sf *= Distributions[i].CCDF(x); - } - else - { - hf += Distributions[i].PDF(x) / Distributions[i].CDF(x); - sf *= Distributions[i].CDF(x); - } + f = PDFMinimumIndependent(x); + } + else + { + f = PDFMaximumIndependent(x); } - f = hf * sf; } else { @@ -514,9 +509,245 @@ public override double PDF(double x) f = NumericalDerivative.Derivative(CDF, x); } - return f < 0d ? 0d : f; + // Return minimum density instead of zero to prevent log-likelihood issues + return f < _minDensity ? _minDensity : f; + } + + /// + /// Computes PDF for minimum of independent random variables. + /// f(x) = h(x) * S(x) where h(x) = sum of hazard rates, S(x) = product of survival functions + /// + private double PDFMinimumIndependent(double x) + { + double sumHazard = 0.0; + double productSurvival = 1.0; + + for (int i = 0; i < Distributions.Count; i++) + { + double ccdf = Distributions[i].CCDF(x); + double pdf = Distributions[i].PDF(x); + + productSurvival *= ccdf; + + // Safe hazard calculation + if (ccdf > _minDensity) + { + sumHazard += pdf / ccdf; + } + else if (pdf > _minDensity) + { + // CCDF ≈ 0 but PDF > 0: we're at the boundary + // The hazard is very large, but productSurvival will be ≈ 0 + // so the contribution is negligible + sumHazard += pdf / _minDensity; // Cap the hazard + } + } + + return sumHazard * productSurvival; + } + + /// + /// Computes PDF for maximum of independent random variables. + /// f(x) = sum_i [f_i(x) * prod_{j≠i} F_j(x)] + /// = [sum_i (f_i/F_i)] * [prod_j F_j] + /// + private double PDFMaximumIndependent(double x) + { + double sumRatio = 0.0; + double productCdf = 1.0; + + for (int i = 0; i < Distributions.Count; i++) + { + double cdf = Distributions[i].CDF(x); + double pdf = Distributions[i].PDF(x); + + productCdf *= cdf; + + // Safe ratio calculation + if (cdf > _minDensity) + { + sumRatio += pdf / cdf; + } + else if (pdf > _minDensity) + { + // CDF ≈ 0 but PDF > 0: we're at the left boundary + // The ratio is very large, but productCdf will be ≈ 0 + // so the contribution is negligible + sumRatio += pdf / _minDensity; // Cap the ratio + } + } + + return sumRatio * productCdf; } + /// + public override double LogPDF(double x) + { + // Validate parameters + if (_parametersValid == false) + ValidateParameters(GetParameters, true); + + if (Distributions.Count == 1) + { + return Distributions[0].LogPDF(x); + } + + // Only compute the exact LogPDF for independent random variables + if (Dependency == Probability.DependencyType.Independent) + { + if (MinimumOfRandomVariables) + { + return LogPDFMinimumIndependent(x); + } + else + { + return LogPDFMaximumIndependent(x); + } + } + else + { + // For dependent cases, fall back to numerical differentiation + // but use a more stable approach + double pdf = NumericalDerivative.Derivative(CDF, x); + return pdf > _minDensity ? Math.Log(pdf) : double.MinValue; + } + + } + + /// + /// Computes log-PDF for minimum of independent random variables. + /// Uses the formula: log(f(x)) = log(sum of hazards) + sum of log(survival functions) + /// + /// For minimum: f(x) = h(x) * S(x) where: + /// h(x) = sum_i h_i(x) = sum_i [f_i(x) / S_i(x)] + /// S(x) = prod_i S_i(x) + /// + /// In log space: log(f) = log(h(x)) + sum_i log(S_i(x)) + /// + private double LogPDFMinimumIndependent(double x) + { + int n = Distributions.Count; + var logSurvival = new double[n]; + var logHazard = new double[n]; + + double sumLogSurvival = 0.0; + bool allSurvivalZero = true; + + for (int i = 0; i < n; i++) + { + double ccdf = Distributions[i].CCDF(x); + double pdf = Distributions[i].PDF(x); + + if (ccdf > _minDensity) + { + logSurvival[i] = Math.Log(ccdf); + allSurvivalZero = false; + } + else + { + // Survival is essentially zero - we're far in the right tail + logSurvival[i] = _logZero; + } + + sumLogSurvival += logSurvival[i]; + + // Compute log-hazard: log(f_i / S_i) = log(f_i) - log(S_i) + if (pdf > _minDensity && ccdf > _minDensity) + { + logHazard[i] = Math.Log(pdf) - Math.Log(ccdf); + } + else if (pdf <= _minDensity) + { + logHazard[i] = _logZero; + } + else + { + // pdf > 0 but ccdf ≈ 0: hazard is very large + // This happens in the far right tail + logHazard[i] = Math.Log(pdf) - _logZero; // Will be very large + } + } + + // If all survival functions are zero, density is zero + if (allSurvivalZero) + return _logZero; + + // Compute log of sum of hazards using log-sum-exp trick + double logSumHazard = Tools.LogSumExp(logHazard); + + // Final result: log(f) = log(sum h_i) + sum log(S_i) + double logPdf = logSumHazard + sumLogSurvival; + + return double.IsNaN(logPdf) || double.IsInfinity(logPdf) ? double.MinValue : logPdf; + } + + /// + /// Computes log-PDF for maximum of independent random variables. + /// Uses the formula: f(x) = sum_i [f_i(x) * prod_{j≠i} F_j(x)] + /// + /// In log space, we use the identity: + /// f(x) = [sum_i (f_i/F_i)] * [prod_j F_j] + /// + /// So: log(f) = log(sum_i f_i/F_i) + sum_j log(F_j) + /// + private double LogPDFMaximumIndependent(double x) + { + int n = Distributions.Count; + var logCdf = new double[n]; + var logRatio = new double[n]; // log(f_i / F_i) + + double sumLogCdf = 0.0; + bool allCdfZero = true; + + for (int i = 0; i < n; i++) + { + double cdf = Distributions[i].CDF(x); + double pdf = Distributions[i].PDF(x); + + if (cdf > _minDensity) + { + logCdf[i] = Math.Log(cdf); + allCdfZero = false; + } + else + { + // CDF is essentially zero - we're far in the left tail + logCdf[i] = _logZero; + } + + sumLogCdf += logCdf[i]; + + // Compute log(f_i / F_i) = log(f_i) - log(F_i) + if (pdf > _minDensity && cdf > _minDensity) + { + logRatio[i] = Math.Log(pdf) - Math.Log(cdf); + } + else if (pdf <= _minDensity) + { + logRatio[i] = _logZero; + } + else + { + // pdf > 0 but cdf ≈ 0: ratio is very large + // This can happen but contributes negligibly when multiplied by near-zero CDF product + logRatio[i] = Math.Log(pdf) - _logZero; + } + } + + // If all CDFs are zero, density is zero + if (allCdfZero) + return _logZero; + + // Compute log of sum of ratios using log-sum-exp trick + double logSumRatio = Tools.LogSumExp(logRatio); + + // Final result: log(f) = log(sum f_i/F_i) + sum log(F_i) + double logPdf = logSumRatio + sumLogCdf; + + return double.IsNaN(logPdf) || double.IsInfinity(logPdf) ? double.MinValue : logPdf; + } + + /// public override double CDF(double x) { @@ -913,6 +1144,84 @@ public override double[] GenerateRandomValues(int sampleSize, int seed = -1) return sample; } + /// + /// Generates random values accounting for dependency structure. + /// The original implementation only handles independent case correctly. + /// + /// Size of random sample to generate. + /// Optional. The prng seed. If negative or zero, then the computer clock is used as a seed. + /// Array of random samples. + public double[] GenerateRandomValuesWithDependency(int sampleSize, int seed = -1) + { + var rnd = seed > 0 ? new MersenneTwister(seed) : new MersenneTwister(); + var sample = new double[sampleSize]; + + if (Dependency == Probability.DependencyType.Independent) + { + // Original implementation is correct for independent case + for (int i = 0; i < sampleSize; i++) + { + double xMin = double.MaxValue; + double xMax = double.MinValue; + for (int j = 0; j < Distributions.Count; j++) + { + var x = Distributions[j].InverseCDF(rnd.NextDouble()); + if (x < xMin) xMin = x; + if (x > xMax) xMax = x; + } + sample[i] = MinimumOfRandomVariables ? xMin : xMax; + } + } + else if (Dependency == Probability.DependencyType.PerfectlyPositive) + { + // For perfectly positive dependency, all variables share the same quantile + for (int i = 0; i < sampleSize; i++) + { + double u = rnd.NextDouble(); + double xMin = double.MaxValue; + double xMax = double.MinValue; + for (int j = 0; j < Distributions.Count; j++) + { + var x = Distributions[j].InverseCDF(u); // Same u for all + if (x < xMin) xMin = x; + if (x > xMax) xMax = x; + } + sample[i] = MinimumOfRandomVariables ? xMin : xMax; + } + } + else if (Dependency == Probability.DependencyType.PerfectlyNegative || + Dependency == Probability.DependencyType.CorrelationMatrix) + { + // Use Gaussian copula for correlation structure + if (_mvnCreated == false) + CreateMultivariateNormal(); + + // Generate correlated standard normal samples + var mvnSamples = _mvn.GenerateRandomValues(sampleSize, seed); + + for (int i = 0; i < sampleSize; i++) + { + double xMin = double.MaxValue; + double xMax = double.MinValue; + + for (int j = 0; j < Distributions.Count; j++) + { + // Transform standard normal to uniform via Phi, then to marginal via inverse CDF + double z = mvnSamples[i, j]; + double u = Normal.StandardCDF(z); + var x = Distributions[j].InverseCDF(u); + + if (x < xMin) xMin = x; + if (x > xMax) xMax = x; + } + sample[i] = MinimumOfRandomVariables ? xMin : xMax; + } + } + + return sample; + } + + /// public override UnivariateDistributionBase Clone() { @@ -928,7 +1237,7 @@ public override UnivariateDistributionBase Clone() ProbabilityTransform = ProbabilityTransform }; if (CorrelationMatrix != null) - cr.CorrelationMatrix = (double[,]) CorrelationMatrix.Clone(); + cr.CorrelationMatrix = (double[,])CorrelationMatrix.Clone(); return cr; } @@ -992,19 +1301,19 @@ public override XElement ToXElement() public static CompetingRisks? FromXElement(XElement xElement) { UnivariateDistributionType type = UnivariateDistributionType.Deterministic; - var xElAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); - if (xElAttr != null) + var typeAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); + if (typeAttr != null) { - Enum.TryParse(xElAttr.Value, out type); + Enum.TryParse(typeAttr.Value, out type); } if (type == UnivariateDistributionType.CompetingRisks) { var distributions = new List(); - var xDistAttr = xElement.Attribute(nameof(Distributions)); - if (xDistAttr != null) + var distsAttr = xElement.Attribute(nameof(Distributions)); + if (distsAttr != null) { - var types = xDistAttr.Value.Split('|'); + var types = distsAttr.Value.Split('|'); for (int i = 0; i < types.Length; i++) { Enum.TryParse(types[i], out UnivariateDistributionType distType); @@ -1013,48 +1322,36 @@ public override XElement ToXElement() } var competingRisks = new CompetingRisks(distributions.ToArray()); - if (xElement.Attribute(nameof(XTransform)) != null) + var xTransformAttr = xElement.Attribute(nameof(XTransform)); + if (xTransformAttr != null) { - var xTransformAttr = xElement.Attribute(nameof(XTransform)); - if (xTransformAttr != null) - { - Enum.TryParse(xTransformAttr.Value, out Transform xTransform); - competingRisks.XTransform = xTransform; - } + Enum.TryParse(xTransformAttr.Value, out Transform xTransform); + competingRisks.XTransform = xTransform; } - if (xElement.Attribute(nameof(ProbabilityTransform)) != null) + var probTransformAttr = xElement.Attribute(nameof(ProbabilityTransform)); + if (probTransformAttr != null) { - var xProbabilityAttr = xElement.Attribute(nameof(ProbabilityTransform)); - if (xProbabilityAttr != null) - { - Enum.TryParse(xProbabilityAttr.Value, out Transform probabilityTransform); - competingRisks.ProbabilityTransform = probabilityTransform; - } + Enum.TryParse(probTransformAttr.Value, out Transform probabilityTransform); + competingRisks.ProbabilityTransform = probabilityTransform; } - if (xElement.Attribute(nameof(MinimumOfRandomVariables)) != null) + var minOfRVAttr = xElement.Attribute(nameof(MinimumOfRandomVariables)); + if (minOfRVAttr != null) { - var xMinOfAttr = xElement.Attribute(nameof(MinimumOfRandomVariables)); - if (xMinOfAttr != null) - { - bool.TryParse(xMinOfAttr.Value, out bool minOfValues); - competingRisks.MinimumOfRandomVariables = minOfValues; - } + bool.TryParse(minOfRVAttr.Value, out bool minOfValues); + competingRisks.MinimumOfRandomVariables = minOfValues; } - if (xElement.Attribute(nameof(Dependency)) != null) + var depAttr = xElement.Attribute(nameof(Dependency)); + if (depAttr != null) { - var xDependencyAttr = xElement.Attribute(nameof(Dependency)); - if (xDependencyAttr != null) - { - Enum.TryParse(xDependencyAttr.Value, out Probability.DependencyType dependency); - competingRisks.Dependency = dependency; - } + Enum.TryParse(depAttr.Value, out Probability.DependencyType dependency); + competingRisks.Dependency = dependency; } // Parameters - var xParametersAttr = xElement.Attribute("Parameters"); - if (xParametersAttr != null) - { - var vals = xParametersAttr.Value.Split('|'); + var paramsAttr = xElement.Attribute("Parameters"); + if (paramsAttr != null) + { + var vals = paramsAttr.Value.Split('|'); var parameters = new List(); for (int i = 0; i < vals.Length; i++) { diff --git a/Numerics/Distributions/Univariate/Deterministic.cs b/Numerics/Distributions/Univariate/Deterministic.cs index 99cdad1e..3549db26 100644 --- a/Numerics/Distributions/Univariate/Deterministic.cs +++ b/Numerics/Distributions/Univariate/Deterministic.cs @@ -229,14 +229,14 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { // Validate probability if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0])) { if (throwException) - throw new ArgumentOutOfRangeException(nameof(Probability), "The point value must be a number."); - return new ArgumentOutOfRangeException(nameof(Probability), "The point value must be a number."); + throw new ArgumentOutOfRangeException(nameof(Value), "The point value must be a number."); + return new ArgumentOutOfRangeException(nameof(Value), "The point value must be a number."); } return null!; } diff --git a/Numerics/Distributions/Univariate/EmpiricalDistribution.cs b/Numerics/Distributions/Univariate/EmpiricalDistribution.cs index af05b7e9..d348321d 100644 --- a/Numerics/Distributions/Univariate/EmpiricalDistribution.cs +++ b/Numerics/Distributions/Univariate/EmpiricalDistribution.cs @@ -155,19 +155,14 @@ public EmpiricalDistribution(IList sample, PlottingPositions.PlottingPos { _xValues = sample.ToArray(); Array.Sort(_xValues); - - var pValues = PlottingPositions.Function(_xValues.Count(), plottingPostionType); - - if (pValues is null) { throw new InvalidOperationException("PlottingPositions.Function returned null."); } - _pValues = pValues; - + _pValues = PlottingPositions.Function(_xValues.Count(), plottingPostionType)!; opd = new OrderedPairedData(_xValues, _pValues, true, SortOrder.Ascending, true, SortOrder.Ascending); _momentsComputed = false; } - private double[] _xValues = Array.Empty(); - private double[] _pValues = Array.Empty(); - private OrderedPairedData opd = default!; + private double[] _xValues = null!; + private double[] _pValues = null!; + private OrderedPairedData opd = null!; private bool _momentsComputed = false; private double u1, u2, u3, u4; @@ -270,7 +265,7 @@ public override string[] GetParameterPropertyNames /// public override double[] GetParameters { - get { throw new NotImplementedException(); } + get { return []; } } @@ -410,9 +405,9 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { - return new ArgumentOutOfRangeException("The parameters are valid"); + return null; } /// @@ -704,7 +699,7 @@ public static EmpiricalDistribution Convolve(IList distri throw new ArgumentException("Distribution list cannot be null or empty.", nameof(distributions)); if (distributions.Count == 1) - return (EmpiricalDistribution) distributions[0].Clone(); + return (EmpiricalDistribution)distributions[0].Clone(); if (numberOfPoints < 2) throw new ArgumentException("Number of points must be at least 2.", nameof(numberOfPoints)); diff --git a/Numerics/Distributions/Univariate/Exponential.cs b/Numerics/Distributions/Univariate/Exponential.cs index 7bf019d1..f35a9165 100644 --- a/Numerics/Distributions/Univariate/Exponential.cs +++ b/Numerics/Distributions/Univariate/Exponential.cs @@ -51,7 +51,7 @@ namespace Numerics.Distributions [Serializable] public sealed class Exponential : UnivariateDistributionBase, IEstimation, IMaximumLikelihoodEstimation, IMomentEstimation, ILinearMomentEstimation, IStandardError, IBootstrappable { - + /// /// Constructs an Exponential distribution with a location of 100 and scale of 10. /// @@ -69,7 +69,15 @@ public Exponential(double location, double scale) { SetParameters(location, scale); } - + + /// + /// Constructs an Exponential distribution with a location of 0 and a given scale. + /// + /// The scale parameter α (alpha). + public Exponential(double scale) : this(0.0d, scale) + { + } + private double _xi; // location private double _alpha; // scale @@ -271,7 +279,7 @@ public override void SetParameters(IList parameters) /// /// A list of parameters. /// Determines whether to throw an exception or not. - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0])) { diff --git a/Numerics/Distributions/Univariate/GammaDistribution.cs b/Numerics/Distributions/Univariate/GammaDistribution.cs index 5b8419a0..a157d3e3 100644 --- a/Numerics/Distributions/Univariate/GammaDistribution.cs +++ b/Numerics/Distributions/Univariate/GammaDistribution.cs @@ -375,7 +375,7 @@ public override void SetParameters(IList parameters) /// The scale parameter θ (theta). /// The shape parameter k. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double scale, double shape, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double scale, double shape, bool throwException) { if (double.IsNaN(scale) || double.IsInfinity(scale) || scale <= 0.0d) { @@ -393,7 +393,7 @@ public ArgumentOutOfRangeException ValidateParameters(double scale, double shape } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } @@ -770,22 +770,60 @@ public override UnivariateDistributionBase Clone() /// public double[,] ParameterCovariance(int sampleSize, ParameterEstimationMethod estimationMethod) { - if (estimationMethod != ParameterEstimationMethod.MaximumLikelihood) + if (estimationMethod != ParameterEstimationMethod.MethodOfMoments && + estimationMethod != ParameterEstimationMethod.MaximumLikelihood) { throw new NotImplementedException(); } // Validate parameters if (_parametersValid == false) ValidateParameters(_theta, _kappa, true); - // Compute covariance - double alpha = 1d / Theta; - double lambda = Kappa; - double NA = Gamma.Trigamma(lambda) - 1d / lambda; + // Compute covariance in user-facing (θ=scale, κ=shape) parameterization. + double t = Theta, k = Kappa; var covar = new double[2, 2]; - covar[0, 0] = Math.Pow(alpha, 2d) * Gamma.Trigamma(lambda) / (sampleSize * lambda * NA); // scale - covar[1, 1] = 1d / (sampleSize * NA); // shape - covar[0, 1] = alpha / (sampleSize * lambda * NA); // scale & shape - covar[1, 0] = covar[0, 1]; + if (estimationMethod == ParameterEstimationMethod.MethodOfMoments) + { + // MoM asymptotic covariance via (DᵀS⁻¹D)⁻¹/n. + // Moment conditions: g₁ = X − κθ, g₂ = (X−κθ)² − κθ². + // D = ∂g/∂(θ,κ) = [[-κ, -θ], [-2κθ, -θ²]]. + // S = E[g·gᵀ] = [[μ₂, μ₃], [μ₃, μ₄−μ₂²]]. + double t2 = t * t, t3 = t2 * t, t4 = t2 * t2; + // Central moments of Gamma(θ, κ) + double mu2 = k * t2; + double mu3 = 2.0 * k * t3; + double mu4 = 3.0 * k * (k + 2.0) * t4; + // S matrix and its inverse + double S00 = mu2, S01 = mu3, S11 = mu4 - mu2 * mu2; + double detS = S00 * S11 - S01 * S01; + double Si00 = S11 / detS, Si01 = -S01 / detS, Si11 = S00 / detS; + // D matrix + double d00 = -k, d01 = -t, d10 = -2.0 * k * t, d11 = -t2; + // DᵀS⁻¹ + double ds00 = d00 * Si00 + d10 * Si01; + double ds01 = d00 * Si01 + d10 * Si11; + double ds10 = d01 * Si00 + d11 * Si01; + double ds11 = d01 * Si01 + d11 * Si11; + // Bread = (DᵀS⁻¹)D + double b00 = ds00 * d00 + ds01 * d10; + double b01 = ds00 * d01 + ds01 * d11; + double b11 = ds10 * d01 + ds11 * d11; + // Bread⁻¹ / n + double detB = b00 * b11 - b01 * b01; + covar[0, 0] = b11 / (detB * sampleSize); + covar[1, 1] = b00 / (detB * sampleSize); + covar[0, 1] = -b01 / (detB * sampleSize); + covar[1, 0] = covar[0, 1]; + } + else + { + // MLE: Fisher information inverse in (θ, κ) space. + // Transformed from (α=1/θ, κ) via delta method. + double NA = Gamma.Trigamma(k) - 1.0 / k; + covar[0, 0] = t * t * Gamma.Trigamma(k) / (sampleSize * k * NA); // Var(θ̂) + covar[1, 1] = 1.0 / (sampleSize * NA); // Var(κ̂) + covar[0, 1] = -t / (sampleSize * k * NA); // Cov(θ̂, κ̂) — negative + covar[1, 0] = covar[0, 1]; + } return covar; } @@ -829,13 +867,11 @@ public double[] QuantileGradient(double probability) // Validate parameters if (_parametersValid == false) ValidateParameters(_theta, _kappa, true); - double alpha = 1d / Theta; - double lambda = Kappa; - double eps = Math.Sign(alpha); + // Q(p) = κθ + √κ·θ·Kp(γ,p) in (θ, κ) parameterization. var gradient = new double[] { - -lambda / Math.Pow(alpha, 2d) * (1.0d + eps / Math.Sqrt(lambda) * FrequencyFactorKp(Skewness, probability)), // scale - 1.0d / alpha * (1.0d + eps / Math.Sqrt(lambda) * FrequencyFactorKp(Skewness, probability) / 2.0d - 1.0d / lambda * PartialKp(Skewness, probability)) // shape + PartialforTheta(probability), // ∂Q/∂θ + PartialforKappa(probability) // ∂Q/∂κ }; return gradient; } diff --git a/Numerics/Distributions/Univariate/GeneralizedBeta.cs b/Numerics/Distributions/Univariate/GeneralizedBeta.cs index 24dd2472..ae9135d7 100644 --- a/Numerics/Distributions/Univariate/GeneralizedBeta.cs +++ b/Numerics/Distributions/Univariate/GeneralizedBeta.cs @@ -271,6 +271,7 @@ public override double Mode { get { + if (Alpha <= 1.0d && Beta <= 1.0d) return (Min + Max) / 2.0d; double _mode = (Alpha - 1.0d) / (Alpha + Beta - 2.0d); return _mode * (Max - Min) + Min; } @@ -378,7 +379,7 @@ public void SetParametersFromMoments(double mu, double sigma, double min, double /// The minimum possible value. /// The maximum possible value. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double alpha, double beta, double min, double max, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double alpha, double beta, double min, double max, bool throwException) { if (double.IsNaN(alpha) || double.IsInfinity(alpha) || alpha <= 0.0d) { @@ -403,7 +404,7 @@ public ArgumentOutOfRangeException ValidateParameters(double alpha, double beta, } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], parameters[3], throwException); } diff --git a/Numerics/Distributions/Univariate/GeneralizedExtremeValue.cs b/Numerics/Distributions/Univariate/GeneralizedExtremeValue.cs index 3d89127e..957a5fb0 100644 --- a/Numerics/Distributions/Univariate/GeneralizedExtremeValue.cs +++ b/Numerics/Distributions/Univariate/GeneralizedExtremeValue.cs @@ -401,7 +401,7 @@ public override void SetParameters(IList parameters) /// The scale parameter α (alpha). /// The shape parameter κ (kappa). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double location, double scale, double shape, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double location, double scale, double shape, bool throwException) { if (double.IsNaN(location) || double.IsInfinity(location)) { @@ -424,7 +424,7 @@ public ArgumentOutOfRangeException ValidateParameters(double location, double sc } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } diff --git a/Numerics/Distributions/Univariate/GeneralizedLogistic.cs b/Numerics/Distributions/Univariate/GeneralizedLogistic.cs index 81093b4a..2066cf7e 100644 --- a/Numerics/Distributions/Univariate/GeneralizedLogistic.cs +++ b/Numerics/Distributions/Univariate/GeneralizedLogistic.cs @@ -393,7 +393,7 @@ public override void SetParameters(IList parameters) /// The scale parameter α (alpha). /// The shape parameter κ (kappa). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double location, double scale, double shape, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double location, double scale, double shape, bool throwException) { if (double.IsNaN(location) || double.IsInfinity(location)) { @@ -416,7 +416,7 @@ public ArgumentOutOfRangeException ValidateParameters(double location, double sc } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } diff --git a/Numerics/Distributions/Univariate/GeneralizedNormal.cs b/Numerics/Distributions/Univariate/GeneralizedNormal.cs index b6888930..c61c69f6 100644 --- a/Numerics/Distributions/Univariate/GeneralizedNormal.cs +++ b/Numerics/Distributions/Univariate/GeneralizedNormal.cs @@ -349,7 +349,7 @@ public override void SetParameters(IList parameters) /// The scale parameter α (alpha). /// The shape parameter κ (kappa). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double location, double scale, double shape, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double location, double scale, double shape, bool throwException) { if (double.IsNaN(location) || double.IsInfinity(location)) { @@ -372,7 +372,7 @@ public ArgumentOutOfRangeException ValidateParameters(double location, double sc } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } diff --git a/Numerics/Distributions/Univariate/GeneralizedPareto.cs b/Numerics/Distributions/Univariate/GeneralizedPareto.cs index bcdb1fac..ed5ecc27 100644 --- a/Numerics/Distributions/Univariate/GeneralizedPareto.cs +++ b/Numerics/Distributions/Univariate/GeneralizedPareto.cs @@ -393,7 +393,7 @@ public override void SetParameters(IList parameters) /// The scale parameter α (alpha). /// The shape parameter κ (kappa). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double location, double scale, double shape, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double location, double scale, double shape, bool throwException) { if (double.IsNaN(location) || double.IsInfinity(location)) { @@ -416,7 +416,7 @@ public ArgumentOutOfRangeException ValidateParameters(double location, double sc } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } diff --git a/Numerics/Distributions/Univariate/Geometric.cs b/Numerics/Distributions/Univariate/Geometric.cs index 7dbe5b60..9fe33ad9 100644 --- a/Numerics/Distributions/Univariate/Geometric.cs +++ b/Numerics/Distributions/Univariate/Geometric.cs @@ -210,7 +210,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { // Validate probability if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0]) || parameters[0] < 0.0d || parameters[0] > 1.0d) diff --git a/Numerics/Distributions/Univariate/Gumbel.cs b/Numerics/Distributions/Univariate/Gumbel.cs index 97b1b428..a2a148a5 100644 --- a/Numerics/Distributions/Univariate/Gumbel.cs +++ b/Numerics/Distributions/Univariate/Gumbel.cs @@ -276,7 +276,7 @@ public override void SetParameters(IList parameters) /// The location parameter ξ (Xi). /// The scale parameter α (alpha). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double location, double scale, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double location, double scale, bool throwException) { if (double.IsNaN(location) || double.IsInfinity(location)) { @@ -294,7 +294,7 @@ public ArgumentOutOfRangeException ValidateParameters(double location, double sc } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } diff --git a/Numerics/Distributions/Univariate/InverseChiSquared.cs b/Numerics/Distributions/Univariate/InverseChiSquared.cs index 433e2b41..f8911d7f 100644 --- a/Numerics/Distributions/Univariate/InverseChiSquared.cs +++ b/Numerics/Distributions/Univariate/InverseChiSquared.cs @@ -255,7 +255,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (parameters[0] < 1.0d) { @@ -298,7 +298,7 @@ public override double CDF(double x) return 0d; double v = DegreesOfFreedom; double t2 = Sigma; - return Gamma.UpperIncomplete(v / 2.0d, v * t2 / 2.0d * x); + return Gamma.UpperIncomplete(v / 2.0d, v * t2 / (2.0d * x)); } /// @@ -316,7 +316,7 @@ public override double InverseCDF(double probability) ValidateParameters([DegreesOfFreedom, Sigma], true); double v = DegreesOfFreedom; double t2 = Sigma; - return Gamma.InverseUpperIncomplete(v / 2.0d, probability) / (t2 * v / 2.0d); + return v * t2 / (2.0d * Gamma.InverseUpperIncomplete(v / 2.0d, probability)); } /// diff --git a/Numerics/Distributions/Univariate/InverseGamma.cs b/Numerics/Distributions/Univariate/InverseGamma.cs index 778b5a65..5d7d6999 100644 --- a/Numerics/Distributions/Univariate/InverseGamma.cs +++ b/Numerics/Distributions/Univariate/InverseGamma.cs @@ -241,7 +241,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0]) || parameters[0] <= 0.0d) { diff --git a/Numerics/Distributions/Univariate/KappaFour.cs b/Numerics/Distributions/Univariate/KappaFour.cs index 390b6306..16bc2f34 100644 --- a/Numerics/Distributions/Univariate/KappaFour.cs +++ b/Numerics/Distributions/Univariate/KappaFour.cs @@ -380,7 +380,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0])) { diff --git a/Numerics/Distributions/Univariate/KernelDensity.cs b/Numerics/Distributions/Univariate/KernelDensity.cs index 7cc8c808..e16a1aed 100644 --- a/Numerics/Distributions/Univariate/KernelDensity.cs +++ b/Numerics/Distributions/Univariate/KernelDensity.cs @@ -152,14 +152,14 @@ public enum KernelType Uniform } - private double[] _sampleData = Array.Empty(); + private double[] _sampleData = null!; private double _bandwidth; private KernelType _kernelDistribution; private IKernel _kernel = null!; private bool _cdfCreated = false; private OrderedPairedData opd = null!; private double u1, u2, u3, u4; - private double[] _weights = null!; // one weight per sample (unnormalised) + private double[]? _weights; // one weight per sample (unnormalised) private double _sumW = 1.0; // Σ wᵢ (defaults to 1 for un‑weighted case) @@ -184,7 +184,7 @@ public KernelType KernelDistribution } else if (_kernelDistribution == KernelType.Gaussian) { - _kernel = new GuassianKernel(); + _kernel = new GaussianKernel(); } else if (_kernelDistribution == KernelType.Triangular) { @@ -299,7 +299,7 @@ public override string[] GetParameterPropertyNames /// public override double[] GetParameters { - get { throw new NotImplementedException(); } + get { return []; } } /// @@ -459,7 +459,7 @@ public double Function(double x) /// Gaussian kernel with a mean of 0 and standard deviation of 1. /// This is the default kernel. /// - private class GuassianKernel : IKernel + private class GaussianKernel : IKernel { /// /// Evaluates the Gaussian (Normal) kernel function at the specified point. @@ -549,7 +549,7 @@ public double BandwidthRule(IList sampleData) /// /// Sample of data, no sorting is assumed. /// A list of weights. - public double BandwidthRule(IList sample, IList w = null!) + public double BandwidthRule(IList sample, IList? w = null) { w ??= Enumerable.Repeat(1.0, sample.Count).ToArray(); double m = w.Zip(sample, (wi, xi) => wi * xi).Sum() / w.Sum(); @@ -572,15 +572,15 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { - return null!; + return null; } /// /// Validate the bandwidth parameter. /// - private ArgumentOutOfRangeException ValidateParameters(double value, bool throwException) + private ArgumentOutOfRangeException? ValidateParameters(double value, bool throwException) { if (value <= 0d) { @@ -588,7 +588,7 @@ private ArgumentOutOfRangeException ValidateParameters(double value, bool throwE throw new ArgumentOutOfRangeException(nameof(Bandwidth), "The bandwidth must be a positive number!"); return new ArgumentOutOfRangeException(nameof(Bandwidth), "The bandwidth must be a positive number!"); } - return null!; + return null; } /// @@ -602,6 +602,11 @@ public void SetSampleData(IList sampleData) _cdfCreated = false; } + /// + /// Set the sample data with associated weights. + /// + /// Sample of data, no sorting is assumed. + /// Weights associated with each data point. public void SetSampleData(IList sampleData, IList weights) { _sampleData = sampleData.ToArray(); diff --git a/Numerics/Distributions/Univariate/LnNormal.cs b/Numerics/Distributions/Univariate/LnNormal.cs index 9bd42ce4..66d41c7c 100644 --- a/Numerics/Distributions/Univariate/LnNormal.cs +++ b/Numerics/Distributions/Univariate/LnNormal.cs @@ -297,7 +297,7 @@ public override void SetParameters(IList parameters) /// Mean. /// Standard deviation. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double mean, double standardDeviation, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double mean, double standardDeviation, bool throwException) { if (double.IsNaN(mean) || double.IsInfinity(mean)) { @@ -315,7 +315,7 @@ public ArgumentOutOfRangeException ValidateParameters(double mean, double standa } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } diff --git a/Numerics/Distributions/Univariate/LogNormal.cs b/Numerics/Distributions/Univariate/LogNormal.cs index f2829d76..00a0e8b0 100644 --- a/Numerics/Distributions/Univariate/LogNormal.cs +++ b/Numerics/Distributions/Univariate/LogNormal.cs @@ -351,7 +351,7 @@ public override void SetParameters(IList parameters) /// The mean (of log). /// The standard deviation (of log). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double mu, double sigma, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double mu, double sigma, bool throwException) { if (double.IsNaN(mu) || double.IsInfinity(mu)) { @@ -369,7 +369,7 @@ public ArgumentOutOfRangeException ValidateParameters(double mu, double sigma, b } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } @@ -634,11 +634,13 @@ public override UnivariateDistributionBase Clone() // Validate parameters if (_parametersValid == false) ValidateParameters(Mu, _sigma, true); - // Compute covariance - double u2 = Sigma; + // Compute covariance in (μ, σ) parameterization of log-space. + // Var(μ̂) = σ²/n, Var(σ̂) = σ²/(2n), Cov = 0. + // Both MoM and MLE give the same result for Normal (UMVUE). + double s2 = Sigma * Sigma; var covar = new double[2, 2]; - covar[0, 0] = Math.Pow(u2, 2d) / sampleSize; // location - covar[1, 1] = 2d * Math.Pow(u2, 4d) / sampleSize; // scale + covar[0, 0] = s2 / sampleSize; // Var(μ̂) + covar[1, 1] = s2 / (2.0 * sampleSize); // Var(σ̂) covar[0, 1] = 0.0; covar[1, 0] = covar[0, 1]; return covar; @@ -665,10 +667,11 @@ public double[] QuantileGradient(double probability) if (_parametersValid == false) ValidateParameters(Mu, _sigma, true); double z = Normal.StandardZ(probability); + // Q(p) = μ + σ·z(p) in log-space, so ∂Q/∂μ = 1, ∂Q/∂σ = z(p). var gradient = new double[] { - 1.0d, // location - z / (2d * Sigma) // scale + 1.0d, // ∂Q/∂μ + z // ∂Q/∂σ }; return gradient; } diff --git a/Numerics/Distributions/Univariate/LogPearsonTypeIII.cs b/Numerics/Distributions/Univariate/LogPearsonTypeIII.cs index bd2ffdd1..7db094c5 100644 --- a/Numerics/Distributions/Univariate/LogPearsonTypeIII.cs +++ b/Numerics/Distributions/Univariate/LogPearsonTypeIII.cs @@ -497,7 +497,7 @@ public override void SetParameters(IList parameters) /// The standard deviation of the log transformed data. /// The skew of the log transformed data. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double mu, double sigma, double gamma, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double mu, double sigma, double gamma, bool throwException) { if (double.IsNaN(mu) || double.IsInfinity(mu)) { @@ -517,23 +517,23 @@ public ArgumentOutOfRangeException ValidateParameters(double mu, double sigma, d throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma must be a number."); return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma must be a number."); } - if (gamma > 5) + if (gamma > 6) { if (throwException) - throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be less than 5."); - return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be less than 5."); + throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be less than 6."); + return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be less than 6."); } - if (gamma < -5) + if (gamma < -6) { if (throwException) - throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be greater than -5."); - return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be greater than -5."); + throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be greater than -6."); + return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be greater than -6."); } return null!; } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } @@ -712,8 +712,8 @@ public Tuple GetParameterConstraints(IList upperVals[1] = double.IsNaN(upperVals[1]) ? 4 : upperVals[1]; // Get bounds of skew - lowerVals[2] = -5d; - upperVals[2] = 5d; + lowerVals[2] = -6d; + upperVals[2] = 6d; // Correct initial value of skew if necessary if (initialVals[2] <= lowerVals[2] || initialVals[2] >= upperVals[2]) { @@ -847,34 +847,88 @@ public double WilsonHilfertyInverseCDF(double probability) /// public override UnivariateDistributionBase Clone() { - return new LogPearsonTypeIII(Mu, Sigma, Gamma); + var clone = new LogPearsonTypeIII(Mu, Sigma, Gamma); + clone.Base = Base; + return clone; } /// public double[,] ParameterCovariance(int sampleSize, ParameterEstimationMethod estimationMethod) { - if (estimationMethod != ParameterEstimationMethod.MaximumLikelihood) + if (estimationMethod != ParameterEstimationMethod.MethodOfMoments && + estimationMethod != ParameterEstimationMethod.MaximumLikelihood) { throw new NotImplementedException(); } // Validate parameters if (_parametersValid == false) ValidateParameters(_mu, _sigma, _gamma, true); - // Compute covariance - double alpha = 1d / Beta; - double lambda = Alpha; - double A = 2d * Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) - 2d / (lambda - 1d) + 1d / Math.Pow(lambda - 1d, 2d); + // Compute covariance in user-facing (μₗ, σₗ, γₗ) parameterization. + // LP3 parameters are the PT3/Gamma parameters of log(X). var covar = new double[3, 3]; - covar[0, 0] = (lambda - 2d) / (sampleSize * A) * (1d / Math.Pow(alpha, 2d)) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) * lambda - 1d); // location - covar[1, 1] = (lambda - 2d) * Math.Pow(alpha, 2d) / (sampleSize * A) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) / (lambda - 2d) - 1d / Math.Pow(lambda - 1d, 2d)); // scale - covar[2, 2] = 2d / (sampleSize * A); // shape - covar[0, 1] = 1d / sampleSize * ((lambda - 2d) / A) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) - 1d / (lambda - 1d)); // location & scale - covar[1, 0] = covar[0, 1]; - covar[0, 2] = (2d - lambda) / (sampleSize * alpha * A * (lambda - 1d)); // location & shape - covar[2, 0] = covar[0, 2]; - covar[1, 2] = alpha / (sampleSize * A * (lambda - 1d)); // scale & shape - covar[2, 1] = covar[1, 2]; + if (estimationMethod == ParameterEstimationMethod.MethodOfMoments) + { + // MoM asymptotic covariance via Cov = D⁻¹·S·D⁻ᵀ / n. + // Identical to PT3 since LP3 MoM works in log-space. + // Moment conditions centered on model mean μ(θ): + // g₁ = X−μ, g₂ = (X−μ)²−σ², g₃ = (X−μ)³−γσ³. + // Jacobian D = ∂E[g]/∂(μ,σ,γ) is lower-triangular: + // D = [[-1, 0, 0], [0, -2σ, 0], [-3σ², -3γσ², -σ³]] + // Note: D[2,0] = E[∂g₃/∂μ] = E[-3(X−μ)²] = -3σ² ≠ 0 because + // g₃ centers on μ(θ), not on x̄. + // D⁻¹ = [[-1, 0, 0], [0, -1/(2σ), 0], [3/σ, 3γ/(2σ²), -1/σ³]] + double s = _sigma, g = _gamma; + double s2 = s * s, s3 = s2 * s, s4 = s2 * s2, s5 = s4 * s, s6 = s4 * s2; + double g2 = g * g, g4 = g2 * g2; + // Central moments for Pearson III family + double mu2 = s2; + double mu3 = g * s3; + double mu4 = s4 * (3.0 + 1.5 * g2); + double mu5 = s5 * g * (10.0 + 3.0 * g2); + double mu6 = s6 * (15.0 + 32.5 * g2 + 7.5 * g4); + // S matrix elements: S = E[g·gᵀ] + double S00 = mu2; + double S01 = mu3; + double S02 = mu4; + double S11 = mu4 - mu2 * mu2; + double S12 = mu5 - mu2 * mu3; + double S22 = mu6 - mu3 * mu3; + // D⁻¹ elements (lower-triangular) + double a = -1.0; // D⁻¹[0,0] + double b = -1.0 / (2.0 * s); // D⁻¹[1,1] + double e = 3.0 / s; // D⁻¹[2,0] — from D[2,0] = -3σ² + double c = 3.0 * g / (2.0 * s2); // D⁻¹[2,1] + double d = -1.0 / s3; // D⁻¹[2,2] + // Cov = D⁻¹ · S · D⁻ᵀ / n (exploit triangular structure) + covar[0, 0] = a * a * S00 / sampleSize; + covar[0, 1] = a * b * S01 / sampleSize; + covar[0, 2] = a * (e * S00 + c * S01 + d * S02) / sampleSize; + covar[1, 1] = b * b * S11 / sampleSize; + covar[1, 2] = b * (e * S01 + c * S11 + d * S12) / sampleSize; + covar[2, 2] = (e * e * S00 + 2.0 * e * c * S01 + 2.0 * e * d * S02 + + c * c * S11 + 2.0 * c * d * S12 + d * d * S22) / sampleSize; + covar[1, 0] = covar[0, 1]; + covar[2, 0] = covar[0, 2]; + covar[2, 1] = covar[1, 2]; + } + else + { + // MLE: Fisher information inverse in internal (μ, α=1/β, λ=4/γ²) parameterization. + // This parameterization matches QuantileGradient for MLE QuantileVariance computation. + double alpha = 1d / Beta; + double lambda = Alpha; + double A = 2d * Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) - 2d / (lambda - 1d) + 1d / Math.Pow(lambda - 1d, 2d); + covar[0, 0] = (lambda - 2d) / (sampleSize * A) * (1d / Math.Pow(alpha, 2d)) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) * lambda - 1d); // location + covar[1, 1] = (lambda - 2d) * Math.Pow(alpha, 2d) / (sampleSize * A) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) / (lambda - 2d) - 1d / Math.Pow(lambda - 1d, 2d)); // scale + covar[2, 2] = 2d / (sampleSize * A); // shape + covar[0, 1] = 1d / sampleSize * ((lambda - 2d) / A) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) - 1d / (lambda - 1d)); // location & scale + covar[1, 0] = covar[0, 1]; + covar[0, 2] = (2d - lambda) / (sampleSize * alpha * A * (lambda - 1d)); // location & shape + covar[2, 0] = covar[0, 2]; + covar[1, 2] = alpha / (sampleSize * A * (lambda - 1d)); // scale & shape + covar[2, 1] = covar[1, 2]; + } return covar; } @@ -916,6 +970,8 @@ public double[] QuantileGradient(double probability) // Validate parameters if (_parametersValid == false) ValidateParameters(Mu, Sigma, Gamma, true); + // Gradient in internal (μ, α=1/β, λ=4/γ²) parameterization. + // Matches ParameterCovariance(MLE) for MLE QuantileVariance computation. double alpha = 1d / Beta; double lambda = Alpha; double eps = Math.Sign(alpha); diff --git a/Numerics/Distributions/Univariate/Logistic.cs b/Numerics/Distributions/Univariate/Logistic.cs index 9b934d01..579bb7c7 100644 --- a/Numerics/Distributions/Univariate/Logistic.cs +++ b/Numerics/Distributions/Univariate/Logistic.cs @@ -267,7 +267,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0])) { diff --git a/Numerics/Distributions/Univariate/Mixture.cs b/Numerics/Distributions/Univariate/Mixture.cs index a782d856..e52d6aa6 100644 --- a/Numerics/Distributions/Univariate/Mixture.cs +++ b/Numerics/Distributions/Univariate/Mixture.cs @@ -79,7 +79,7 @@ public Mixture(double[] weights, IUnivariateDistribution[] distributions) SetParameters(weights, distributions); } - private double[] _weights = Array.Empty(); + private double[] _weights = null!; private UnivariateDistributionBase[] _distributions = null!; private EmpiricalDistribution _empiricalCDF = null!; private bool _momentsComputed = false; @@ -161,11 +161,11 @@ public override int NumberOfParameters var parmString = new string[2, 2]; string Wstring = "{"; string Dstring = "{"; - for (int i = 1; i < Weights.Count() - 1; i++) + for (int i = 0; i < Weights.Count(); i++) { Wstring += Weights[i].ToString(); Dstring += Distributions[i].DisplayName; - if (i < Weights.Count() - 2) + if (i < Weights.Count() - 1) { Wstring += ","; Dstring += ","; @@ -580,7 +580,7 @@ public void SetParameters(ref double[] parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { // Check if weights are between 0 and 1. if (IsZeroInflated && (ZeroWeight < 0.0 || ZeroWeight > 1.0)) @@ -618,7 +618,7 @@ public override ArgumentOutOfRangeException ValidateParameters(IList par return new ArgumentOutOfRangeException(nameof(Distributions), "Distribution " + (i + 1).ToString() + " has invalid parameters."); } } - return null!; + return null; } /// @@ -761,7 +761,8 @@ double logLH(double[] x) // Perform the expectation step newLogLH = EStep(mleParameters); - // Check convergence + // Check convergence before M-step to avoid pushing parameters + // into a degenerate state after the log-likelihood has already converged. if (Math.Abs((oldLogLH - newLogLH) / oldLogLH) < Tolerance) break; @@ -913,8 +914,8 @@ public override double InverseCDF(double probability) if (_parametersValid == false) ValidateParameters(GetParameters, true); - // If there is only one distribution, return its inverse CDF - if (Distributions.Count() == 1) + // If there is only one distribution and not zero-inflated, return its inverse CDF + if (Distributions.Count() == 1 && !IsZeroInflated) { return Distributions[0].InverseCDF(probability); } @@ -927,9 +928,13 @@ public override double InverseCDF(double probability) else { // Use a root finder to solve the inverse CDF - var xVals = Distributions.Select(d => d.InverseCDF(probability)); - double minX = xVals.Min(); - double maxX = xVals.Max(); + // For zero-inflated mixtures, use adjusted probability for lower bracket + // and unadjusted for upper bracket to ensure the root is bracketed + double adjProb = IsZeroInflated ? (probability - ZeroWeight) / (1d - ZeroWeight) : probability; + var minXVals = Distributions.Select(d => d.InverseCDF(adjProb)); + var maxXVals = Distributions.Select(d => d.InverseCDF(probability)); + double minX = minXVals.Min(); + double maxX = maxXVals.Max(); try { if (IsZeroInflated) @@ -1083,33 +1088,33 @@ public override XElement ToXElement() /// /// The XElement to deserialize. /// A new mixture distribution. - public static Mixture FromXElement(XElement xElement) + public static Mixture? FromXElement(XElement xElement) { UnivariateDistributionType type = UnivariateDistributionType.Deterministic; - var univBaseAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); - if (univBaseAttr != null) + var typeAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); + if (typeAttr != null) { - Enum.TryParse(univBaseAttr.Value, out type); + Enum.TryParse(typeAttr.Value, out type); } if (type == UnivariateDistributionType.Mixture) { var weights = new List(); var distributions = new List(); - var weightAttr = xElement.Attribute(nameof(Weights)); - if (weightAttr != null) + var weightsAttr = xElement.Attribute(nameof(Weights)); + if (weightsAttr != null) { - var w = weightAttr.Value.Split('|'); + var w = weightsAttr.Value.Split('|'); for (int i = 0; i < w.Length; i++) { double.TryParse(w[i], NumberStyles.Any, CultureInfo.InvariantCulture, out var weight); weights.Add(weight); } } - var distAttr = xElement.Attribute(nameof(Distributions)); - if (distAttr != null) + var distsAttr = xElement.Attribute(nameof(Distributions)); + if (distsAttr != null) { - var types = distAttr.Value.Split('|'); + var types = distsAttr.Value.Split('|'); for (int i = 0; i < types.Length; i++) { Enum.TryParse(types[i], out UnivariateDistributionType distType); @@ -1117,6 +1122,7 @@ public static Mixture FromXElement(XElement xElement) } } var mixture = new Mixture(weights.ToArray(), distributions.ToArray()); + var zeroInflatedAttr = xElement.Attribute(nameof(IsZeroInflated)); if (zeroInflatedAttr != null) { @@ -1135,16 +1141,16 @@ public static Mixture FromXElement(XElement xElement) Enum.TryParse(xTransformAttr.Value, out Transform xTransform); mixture.XTransform = xTransform; } - var xProbabilityTransformAttr = xElement.Attribute(nameof(ProbabilityTransform)); - if (xProbabilityTransformAttr != null) + var probTransformAttr = xElement.Attribute(nameof(ProbabilityTransform)); + if (probTransformAttr != null) { - Enum.TryParse(xProbabilityTransformAttr.Value, out Transform probabilityTransform); + Enum.TryParse(probTransformAttr.Value, out Transform probabilityTransform); mixture.ProbabilityTransform = probabilityTransform; } - var xParametersAttr = xElement.Attribute("Parameters"); - if (xParametersAttr != null) + var paramsAttr = xElement.Attribute("Parameters"); + if (paramsAttr != null) { - var vals = xParametersAttr.Value.Split('|'); + var vals = paramsAttr.Value.Split('|'); var parameters = new List(); for (int i = 0; i < vals.Length; i++) { @@ -1158,7 +1164,7 @@ public static Mixture FromXElement(XElement xElement) } else { - return null!; + return null; } } diff --git a/Numerics/Distributions/Univariate/NoncentralT.cs b/Numerics/Distributions/Univariate/NoncentralT.cs index e2b473fa..e7b9cd0c 100644 --- a/Numerics/Distributions/Univariate/NoncentralT.cs +++ b/Numerics/Distributions/Univariate/NoncentralT.cs @@ -31,6 +31,7 @@ using System; using System.Collections.Generic; using Numerics.Mathematics.Optimization; +using Numerics.Mathematics.RootFinding; using Numerics.Mathematics.SpecialFunctions; namespace Numerics.Distributions @@ -73,13 +74,13 @@ public NoncentralT(double degreesOfFreedom, double noncentrality) SetParameters(degreesOfFreedom, noncentrality); } - private int _degreesOfFreedom; + private double _degreesOfFreedom; private double _noncentrality; /// /// Gets and sets the degrees of freedom ν (nu) of the distribution. /// - public int DegreesOfFreedom + public double DegreesOfFreedom { get { return _degreesOfFreedom; } set @@ -259,7 +260,7 @@ public override double[] MaximumOfParameters /// The noncentrality parameter μ (mu). public void SetParameters(double v, double mu) { - DegreesOfFreedom = (int)v; + DegreesOfFreedom = v; Noncentrality = mu; } @@ -275,7 +276,7 @@ public override void SetParameters(IList parameters) /// The degrees of freedom ν (nu). Range: ν > 0. /// The noncentrality parameter μ (mu). /// - public ArgumentOutOfRangeException ValidateParameters(double v, double mu, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double v, double mu, bool throwException) { if (v < 1.0d) { @@ -293,7 +294,7 @@ public ArgumentOutOfRangeException ValidateParameters(double v, double mu, bool } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } @@ -359,27 +360,19 @@ public override double InverseCDF(double probability) /// The noncentrality parameter. private double NCT_CDF(double t, double df, double delta) { - double Z; - double ANS = 0d; + double ANS; try { ANS = NCTDist(t, df, delta); } - catch (Exception ex) + catch (ArgumentException) { - // If the solver fails, then use approximation - if (ex.ToString() == "Maximum number of iterations reached.") - { - Z = (t * (1.0d - 1.0d / (4.0d * df)) - delta) / Math.Sqrt(1.0d + Math.Pow(t, 2d) / (2.0d * df)); - ANS = Normal.StandardCDF(Z); - } - - if (ANS < 0d) - ANS = 0d; - if (ANS > 1d) - ANS = 1d; + // If the series fails to converge, use normal approximation + double Z = (t * (1.0d - 1.0d / (4.0d * df)) - delta) / Math.Sqrt(1.0d + t * t / (2.0d * df)); + ANS = Normal.StandardCDF(Z); + if (ANS < 0d) ANS = 0d; + if (ANS > 1d) ANS = 1d; } - return ANS; } @@ -436,8 +429,8 @@ private double NCTDist(double t, double df, double delta) // // Note - ITRMAX and ERRMAX may be changed to suit one's needs. // - const int ITRMAX = 1000; - const double Errmax = 0.0000001d; + const int ITRMAX = 10000; + const double Errmax = 0.000000001d; // DATA ITRMAX/100.1/, ERRMAX/1.E-06/ // @@ -498,7 +491,7 @@ private double NCTDist(double t, double df, double delta) TNC = TNC + P * XODD + q * XEVEN; ERRBD = two * s * (XODD - GODD); } - while (ERRBD > Errmax & N <= ITRMAX); + while (ERRBD > Errmax && N <= ITRMAX); // Twenty: ; @@ -555,7 +548,7 @@ private double NCT_INV(double p, double df, double delta) t1 = t0 + tInc; y1 = NCTDist(t1, df, delta) - p; iter = 0; - while (y0 < 0d != y1 > 0d & Math.Abs(t1 - t0) > xtol & iter < iterMax) + while ((y0 < 0d) != (y1 > 0d) && Math.Abs(t1 - t0) > xtol && iter < iterMax) { // // Use secant method to extrapolate a zero, but overshoot by w >= 1. @@ -574,182 +567,35 @@ private double NCT_INV(double p, double df, double delta) iter = iter + 1; } // Solve for T using Brent - double ANS = ZBrent(t0, t1, y0, y1, xtol, df, delta, p); + double ANS = Brent.Solve(x => NCTDist(x, df, delta) - p, Math.Min(t0, t1), Math.Max(t0, t1), xtol, reportFailure: false); return ANS; } private double NCTInv0(double P, double N, double D) { - // // Approximates percentage points of the non-central t distribution. // P is the percentage, N is the degrees of freedom, D is the non-centrality parameter. - // - // Non-central t is the ratio (U + D)/(Chi/Sqrt(n)) where U and Chi are independent - // random variables distributed as Normal(0, 1) and Chi(n), respectively. - // D is the non-centrality parameter. When equal to 0, TInvNC is Student's t. - // // Source: Johnson & Kotz, Continuous Univariate Distributions, Volume 2. - // - // NB: The number of iterations appears to go quadratically in D. Thus, for large - // D, we will be in trouble. - // - // VB version (c) 2001 Quantitative Decisions. All rights reserved. - // Contact William A. Huber. www.quantdec.com - // - double b; - double z; - double b2; - double u2; - double T; - // - // Establish entry conditions. - // - // If N < 1.0# Or P <= 0.000001 Or P >= 0.999999 Then - // Return Double.NaN - // End If - - var StudentT = new StudentT(N); - z = Normal.StandardZ(P); - // + double z = Normal.StandardZ(P); + // // Jennett & Welch approximation, formula (14.1). // Intended for large values of D^2, such as are used for most tolerance interval calculations. - // - b = Math.Exp(Gamma.LogGamma((N + 1d) / 2d) - Gamma.LogGamma(N / 2d)) * Math.Sqrt(2d / N); - u2 = z * z; - b2 = b * b; - T = b2 + (1d - b2) * (Math.Pow(D, 2d) - u2); - if (T > 0d) + // + double b = Math.Exp(Gamma.LogGamma((N + 1d) / 2d) - Gamma.LogGamma(N / 2d)) * Math.Sqrt(2d / N); + double u2 = z * z; + double b2 = b * b; + double denom = b2 - u2 * (1d - b2); + double disc = b2 + (1d - b2) * (D * D - u2); + if (disc > 0d && Math.Abs(denom) > 1e-12) { - T = (D * b + z * Math.Sqrt(T)) / (b2 - u2 * (1d - b2)); + return (D * b + z * Math.Sqrt(disc)) / denom; } else { - T = 0d; - } - - return T; - } - - private double ZBrent(double X1, double X2, double y1, double y2, double Tol, double N, double Dnc, double Perc) - { - double ZBrentRet = default; - // - // Finds the zero of NCTDist(x, n, d) - p given that [x1, x2] brackets the zero and - // Y1 = value at X1, Y2 = value at X2. - // - // Translated from Numerical Recipes (1986). - // William A. Huber, 24 March 2001. - // - double a; - double b; - var c = default(double); - double fc; - var D = default(double); - var e = default(double); - double tol1; - double xm; - double s; - double P; - double q; - double r; - double fa; - double fb; - int iter; - const int itmax = 100; - const double eps = 0.00000003d; - a = X1; - b = X2; - fa = y1; - fb = y2; - if (fb * fa > 0d) - { - throw new ArgumentException("Brent's method failed because the root is not bracketed."); + // Fallback: offset central t quantile by the noncentrality parameter + var st = new StudentT(N); + return st.InverseCDF(P) + D; } - - fc = fb; - for (iter = 1; iter <= itmax; iter++) - { - if (fb * fc > 0d) - { - c = a; - fc = fa; - D = b - a; - e = D; - } - - if (Math.Abs(fc) < Math.Abs(fb)) - { - a = b; - b = c; - c = a; - fa = fb; - fb = fc; - fc = fa; - } - - tol1 = 2.0d * eps * Math.Abs(b) + 0.5d * Tol; - xm = 0.5d * (c - b); - if (Math.Abs(xm) <= tol1 | fb == 0d) - { - return b; - } - - if (Math.Abs(e) >= tol1 & Math.Abs(fa) > Math.Abs(fb)) - { - s = fb / fa; - if (a == c) - { - P = 2.0d * xm * s; - q = 1.0d - s; - } - else - { - q = fa / fc; - r = fb / fc; - P = s * (2.0d * xm * q * (q - r) - (b - a) * (r - 1.0d)); - q = (q - 1.0d) * (r - 1.0d) * (s - 1.0d); - } - - if (P > 0d) - q = -q; - P = Math.Abs(P); - if (2.0d * P < 3.0d * xm * q - Math.Abs(tol1 * q) & 2.0d * P < Math.Abs(e * q)) - { - e = D; - D = P / q; - } - else - { - D = xm; - e = D; - } - } - else - { - D = xm; - e = D; - } - - a = b; - fa = fb; - if (Math.Abs(D) > tol1) - { - b = b + D; - } - else if (xm > 0d) // b = b + SIGN(tol1, xm) - { - b = b + Math.Abs(tol1); - } - else - { - b = b - Math.Abs(tol1); - } - - fb = NCTDist(b, N, Dnc) - Perc; - } - - ZBrentRet = b; - return ZBrentRet; } /// diff --git a/Numerics/Distributions/Univariate/Normal.cs b/Numerics/Distributions/Univariate/Normal.cs index 5a993176..cb25df2e 100644 --- a/Numerics/Distributions/Univariate/Normal.cs +++ b/Numerics/Distributions/Univariate/Normal.cs @@ -335,7 +335,7 @@ public double[] LinearMomentsFromParameters(IList parameters) /// The location parameter µ (Mu). /// The scale parameter σ (sigma). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double location, double scale, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double location, double scale, bool throwException) { if (double.IsNaN(location) || double.IsInfinity(location)) { @@ -353,7 +353,7 @@ public ArgumentOutOfRangeException ValidateParameters(double location, double sc } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } @@ -834,11 +834,13 @@ public override UnivariateDistributionBase Clone() { throw new NotImplementedException(); } - // Compute covariance - double u2 = Sigma; + // Compute covariance in (μ, σ) parameterization. + // Var(μ̂) = σ²/n, Var(σ̂) = σ²/(2n), Cov = 0. + // Both MoM and MLE give the same result for Normal (UMVUE). + double s2 = Sigma * Sigma; var covar = new double[2, 2]; - covar[0, 0] = Math.Pow(u2, 2d) / sampleSize; // location - covar[1, 1] = 2d * Math.Pow(u2, 4d) / sampleSize; // scale + covar[0, 0] = s2 / sampleSize; // Var(μ̂) + covar[1, 1] = s2 / (2.0 * sampleSize); // Var(σ̂) covar[0, 1] = 0.0; covar[1, 0] = covar[0, 1]; return covar; @@ -863,12 +865,12 @@ public double[] QuantileGradient(double probability) // Validate parameters if (_parametersValid == false) ValidateParameters(Mu, _sigma, true); - double u2 = Sigma; double z = StandardZ(probability); + // Q(p) = μ + σ·z(p), so ∂Q/∂μ = 1, ∂Q/∂σ = z(p). var gradient = new double[] { - 1.0d, // location - z / (2d * u2) // scale + 1.0d, // ∂Q/∂μ + z // ∂Q/∂σ }; return gradient; } diff --git a/Numerics/Distributions/Univariate/Pareto.cs b/Numerics/Distributions/Univariate/Pareto.cs index e1e0a85e..eee28107 100644 --- a/Numerics/Distributions/Univariate/Pareto.cs +++ b/Numerics/Distributions/Univariate/Pareto.cs @@ -246,7 +246,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0]) || parameters[0] <= 0.0d) { diff --git a/Numerics/Distributions/Univariate/PearsonTypeIII.cs b/Numerics/Distributions/Univariate/PearsonTypeIII.cs index 74a67c67..2cb1a8a8 100644 --- a/Numerics/Distributions/Univariate/PearsonTypeIII.cs +++ b/Numerics/Distributions/Univariate/PearsonTypeIII.cs @@ -374,7 +374,7 @@ public override void SetParameters(IList parameters) /// The standard deviation of the distribution. /// The skew of the distribution. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double mu, double sigma, double gamma, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double mu, double sigma, double gamma, bool throwException) { if (double.IsNaN(mu) || double.IsInfinity(mu)) { @@ -394,23 +394,23 @@ public ArgumentOutOfRangeException ValidateParameters(double mu, double sigma, d throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma must be a number."); return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma must be a number."); } - if (gamma > 5) + if (gamma > 6) { if (throwException) - throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be less than 5."); - return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be less than 5."); + throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be less than 6."); + return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be less than 6."); } - if (gamma < -5) + if (gamma < -6) { if (throwException) - throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be greater than -5."); - return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be greater than -5."); + throw new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be greater than -6."); + return new ArgumentOutOfRangeException(nameof(Gamma), "Gamma = " + gamma + ". Gamma must be greater than -6."); } return null!; } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } @@ -527,8 +527,8 @@ public Tuple GetParameterConstraints(IList lowerVals[1] = Tools.DoubleMachineEpsilon; upperVals[1] = Math.Pow(10d, Math.Ceiling(Math.Log10(initialVals[1]) + 1d)); // Get bounds of skew - lowerVals[2] = -5d; - upperVals[2] = 5d; + lowerVals[2] = -6d; + upperVals[2] = 6d; // Correct initial value of skew if necessary if (initialVals[2] <= lowerVals[2] || initialVals[2] >= upperVals[2]) @@ -673,27 +673,77 @@ public override UnivariateDistributionBase Clone() /// public double[,] ParameterCovariance(int sampleSize, ParameterEstimationMethod estimationMethod) { - if (estimationMethod != ParameterEstimationMethod.MaximumLikelihood) + if (estimationMethod != ParameterEstimationMethod.MethodOfMoments && + estimationMethod != ParameterEstimationMethod.MaximumLikelihood) { throw new NotImplementedException(); } // Validate parameters if (_parametersValid == false) ValidateParameters(_mu, _sigma, _gamma, true); - // Compute covariance - double alpha = 1d / Beta; - double lambda = Alpha; - double A = 2d * Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) - 2d / (lambda - 1d) + 1d / Math.Pow(lambda - 1d, 2d); + // Compute covariance in user-facing (μ, σ, γ) parameterization. var covar = new double[3, 3]; - covar[0, 0] = (lambda - 2d) / (sampleSize * A) * (1d / Math.Pow(alpha, 2d)) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) * lambda - 1d); // location - covar[1, 1] = (lambda - 2d) * Math.Pow(alpha, 2d) / (sampleSize * A) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) / (lambda - 2d) - 1d / Math.Pow(lambda - 1d, 2d)); // scale - covar[2, 2] = 2d / (sampleSize * A); // shape - covar[0, 1] = 1d / sampleSize * ((lambda - 2d) / A) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) - 1d / (lambda - 1d)); // location & scale - covar[1, 0] = covar[0, 1]; - covar[0, 2] = (2d - lambda) / (sampleSize * alpha * A * (lambda - 1d)); // location & shape - covar[2, 0] = covar[0, 2]; - covar[1, 2] = alpha / (sampleSize * A * (lambda - 1d)); // scale & shape - covar[2, 1] = covar[1, 2]; + if (estimationMethod == ParameterEstimationMethod.MethodOfMoments) + { + // MoM asymptotic covariance via Cov = D⁻¹·S·D⁻ᵀ / n. + // Moment conditions centered on model mean μ(θ): + // g₁ = X−μ, g₂ = (X−μ)²−σ², g₃ = (X−μ)³−γσ³. + // Jacobian D = ∂E[g]/∂(μ,σ,γ) is lower-triangular: + // D = [[-1, 0, 0], [0, -2σ, 0], [-3σ², -3γσ², -σ³]] + // Note: D[2,0] = E[∂g₃/∂μ] = E[-3(X−μ)²] = -3σ² ≠ 0 because + // g₃ centers on μ(θ), not on x̄. + // D⁻¹ = [[-1, 0, 0], [0, -1/(2σ), 0], [3/σ, 3γ/(2σ²), -1/σ³]] + double s = _sigma, g = _gamma; + double s2 = s * s, s3 = s2 * s, s4 = s2 * s2, s5 = s4 * s, s6 = s4 * s2; + double g2 = g * g, g4 = g2 * g2; + // Central moments for Pearson III family + double mu2 = s2; + double mu3 = g * s3; + double mu4 = s4 * (3.0 + 1.5 * g2); + double mu5 = s5 * g * (10.0 + 3.0 * g2); + double mu6 = s6 * (15.0 + 32.5 * g2 + 7.5 * g4); + // S matrix elements: S = E[g·gᵀ] + double S00 = mu2; + double S01 = mu3; + double S02 = mu4; + double S11 = mu4 - mu2 * mu2; + double S12 = mu5 - mu2 * mu3; + double S22 = mu6 - mu3 * mu3; + // D⁻¹ elements (lower-triangular) + double a = -1.0; // D⁻¹[0,0] + double b = -1.0 / (2.0 * s); // D⁻¹[1,1] + double e = 3.0 / s; // D⁻¹[2,0] — from D[2,0] = -3σ² + double c = 3.0 * g / (2.0 * s2); // D⁻¹[2,1] + double d = -1.0 / s3; // D⁻¹[2,2] + // Cov = D⁻¹ · S · D⁻ᵀ / n (exploit triangular structure) + covar[0, 0] = a * a * S00 / sampleSize; + covar[0, 1] = a * b * S01 / sampleSize; + covar[0, 2] = a * (e * S00 + c * S01 + d * S02) / sampleSize; + covar[1, 1] = b * b * S11 / sampleSize; + covar[1, 2] = b * (e * S01 + c * S11 + d * S12) / sampleSize; + covar[2, 2] = (e * e * S00 + 2.0 * e * c * S01 + 2.0 * e * d * S02 + + c * c * S11 + 2.0 * c * d * S12 + d * d * S22) / sampleSize; + covar[1, 0] = covar[0, 1]; + covar[2, 0] = covar[0, 2]; + covar[2, 1] = covar[1, 2]; + } + else + { + // MLE: Fisher information inverse in internal (μ, α=1/β, λ=4/γ²) parameterization. + // This parameterization matches QuantileGradient for MLE QuantileVariance computation. + double alpha = 1d / Beta; + double lambda = Alpha; + double A = 2d * Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) - 2d / (lambda - 1d) + 1d / Math.Pow(lambda - 1d, 2d); + covar[0, 0] = (lambda - 2d) / (sampleSize * A) * (1d / Math.Pow(alpha, 2d)) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) * lambda - 1d); // location + covar[1, 1] = (lambda - 2d) * Math.Pow(alpha, 2d) / (sampleSize * A) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) / (lambda - 2d) - 1d / Math.Pow(lambda - 1d, 2d)); // scale + covar[2, 2] = 2d / (sampleSize * A); // shape + covar[0, 1] = 1d / sampleSize * ((lambda - 2d) / A) * (Mathematics.SpecialFunctions.Gamma.Trigamma(lambda) - 1d / (lambda - 1d)); // location & scale + covar[1, 0] = covar[0, 1]; + covar[0, 2] = (2d - lambda) / (sampleSize * alpha * A * (lambda - 1d)); // location & shape + covar[2, 0] = covar[0, 2]; + covar[1, 2] = alpha / (sampleSize * A * (lambda - 1d)); // scale & shape + covar[2, 1] = covar[1, 2]; + } return covar; } @@ -733,6 +783,8 @@ public double[] QuantileGradient(double probability) // Validate parameters if (_parametersValid == false) ValidateParameters(Mu, Sigma, Gamma, true); + // Gradient in internal (μ, α=1/β, λ=4/γ²) parameterization. + // Matches ParameterCovariance(MLE) for MLE QuantileVariance computation. double alpha = 1d / Beta; double lambda = Alpha; double eps = Math.Sign(alpha); diff --git a/Numerics/Distributions/Univariate/Pert.cs b/Numerics/Distributions/Univariate/Pert.cs index 62235836..74830aa8 100644 --- a/Numerics/Distributions/Univariate/Pert.cs +++ b/Numerics/Distributions/Univariate/Pert.cs @@ -327,7 +327,7 @@ public override void SetParameters(IList parameters) /// The mode, or most likely, value of the distribution. /// The maximum possible value of the distribution. /// Determines whether to throw an exception or not. - private ArgumentOutOfRangeException ValidateParameters(double min, double mode, double max, bool throwException) + private ArgumentOutOfRangeException? ValidateParameters(double min, double mode, double max, bool throwException) { if (double.IsNaN(min) || double.IsInfinity(min) || double.IsNaN(max) || double.IsInfinity(max) || min > max) @@ -344,7 +344,7 @@ private ArgumentOutOfRangeException ValidateParameters(double min, double mode, } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } diff --git a/Numerics/Distributions/Univariate/PertPercentile.cs b/Numerics/Distributions/Univariate/PertPercentile.cs index 08d2c83a..b19060c5 100644 --- a/Numerics/Distributions/Univariate/PertPercentile.cs +++ b/Numerics/Distributions/Univariate/PertPercentile.cs @@ -313,7 +313,7 @@ public override void SetParameters(IList parameters) /// The 50th percentile value of the distribution. /// The 95th percentile value of the distribution. /// Determines whether to throw an exception or not. - private ArgumentOutOfRangeException ValidateParameters(double fifth, double fiftieth, double ninetyFifth, bool throwException) + private ArgumentOutOfRangeException? ValidateParameters(double fifth, double fiftieth, double ninetyFifth, bool throwException) { if (double.IsNaN(fifth) || double.IsInfinity(fifth) || double.IsNaN(ninetyFifth) || double.IsInfinity(ninetyFifth) || fifth > ninetyFifth) @@ -326,11 +326,11 @@ private ArgumentOutOfRangeException ValidateParameters(double fifth, double fift if (throwException) throw new ArgumentOutOfRangeException(nameof(Percentile50th), "The 50% must be between the 5% and 95%."); return new ArgumentOutOfRangeException(nameof(Percentile50th), "The 50% must be between the 5% and 95%."); } - return null!; + return null; } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } @@ -504,22 +504,22 @@ public override XElement ToXElement() /// /// The XElement to deserialize. /// A new mixture distribution. - public static PertPercentile FromXElement(XElement xElement) + public static PertPercentile? FromXElement(XElement xElement) { UnivariateDistributionType type = UnivariateDistributionType.Deterministic; - var xUnivAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); - if ( xUnivAttr != null) + var typeAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); + if (typeAttr != null) { - Enum.TryParse(xUnivAttr.Value, out type); + Enum.TryParse(typeAttr.Value, out type); } if (type == UnivariateDistributionType.PertPercentile) { bool parametersSolved = false; - var xParamSolvedAttr = xElement.Attribute("ParametersSolved"); - if (xParamSolvedAttr != null) + var paramsSolvedAttr = xElement.Attribute("ParametersSolved"); + if (paramsSolvedAttr != null) { - bool.TryParse(xParamSolvedAttr.Value, out parametersSolved); + bool.TryParse(paramsSolvedAttr.Value, out parametersSolved); } else { @@ -530,10 +530,10 @@ public static PertPercentile FromXElement(XElement xElement) var vals = new double[dist.NumberOfParameters]; for (int i = 0; i < dist.NumberOfParameters; i++) { - var xAttr = xElement.Attribute(names[i]); - if (xAttr != null) + var nameAttr = xElement.Attribute(names[i]); + if (nameAttr != null) { - double.TryParse(xAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out vals[i]); + double.TryParse(nameAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out vals[i]); } } dist.SetParameters(vals); @@ -542,10 +542,10 @@ public static PertPercentile FromXElement(XElement xElement) } var beta = new GeneralizedBeta(); - var xBetaAttr = xElement.Attribute("BetaParameters"); - if (xBetaAttr != null) + var betaParamsAttr = xElement.Attribute("BetaParameters"); + if (betaParamsAttr != null) { - var vals = xBetaAttr.Value.Split('|'); + var vals = betaParamsAttr.Value.Split('|'); var parameters = new List(); for (int i = 0; i < vals.Length; i++) { @@ -555,28 +555,28 @@ public static PertPercentile FromXElement(XElement xElement) beta.SetParameters(parameters); } double _5th = 0, _50th = 0, _95th = 0; - var xParamAttr = xElement.Attribute("Parameters"); - if (xParamAttr != null) + var paramsAttr = xElement.Attribute("Parameters"); + if (paramsAttr != null) { - var vals = xParamAttr.Value.Split('|'); + var vals = paramsAttr.Value.Split('|'); double.TryParse(vals[0], NumberStyles.Any, CultureInfo.InvariantCulture, out _5th); double.TryParse(vals[1], NumberStyles.Any, CultureInfo.InvariantCulture, out _50th); double.TryParse(vals[2], NumberStyles.Any, CultureInfo.InvariantCulture, out _95th); } - var pert = new PertPercentile() - { - _parametersSolved = parametersSolved, - _beta = beta, - _5th = _5th, - _50th = _50th, + var pert = new PertPercentile() + { + _parametersSolved = parametersSolved, + _beta = beta, + _5th = _5th, + _50th = _50th, _95th = _95th }; return pert; } else { - return null!; + return null; } } diff --git a/Numerics/Distributions/Univariate/PertPercentileZ.cs b/Numerics/Distributions/Univariate/PertPercentileZ.cs index b32bd60f..fa9d2e56 100644 --- a/Numerics/Distributions/Univariate/PertPercentileZ.cs +++ b/Numerics/Distributions/Univariate/PertPercentileZ.cs @@ -302,7 +302,7 @@ public override void SetParameters(IList parameters) /// The 50th percentile value of the distribution. /// The 95th percentile value of the distribution. /// Determines whether to throw an exception or not. - private ArgumentOutOfRangeException ValidateParameters(double fifth, double fiftieth, double ninetyFifth, bool throwException) + private ArgumentOutOfRangeException? ValidateParameters(double fifth, double fiftieth, double ninetyFifth, bool throwException) { if (double.IsNaN(fifth) || double.IsInfinity(fifth) || double.IsNaN(ninetyFifth) || double.IsInfinity(ninetyFifth) || fifth > ninetyFifth) @@ -330,11 +330,11 @@ private ArgumentOutOfRangeException ValidateParameters(double fifth, double fift if (throwException) throw new ArgumentOutOfRangeException(nameof(Percentile95th), "The percentiles must be between 0 and 1."); return new ArgumentOutOfRangeException(nameof(Percentile95th), "The percentiles must be between 0 and 1."); } - return null!; + return null; } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } @@ -501,22 +501,22 @@ public override XElement ToXElement() /// /// The XElement to deserialize. /// A new mixture distribution. - public static PertPercentileZ FromXElement(XElement xElement) + public static PertPercentileZ? FromXElement(XElement xElement) { UnivariateDistributionType type = UnivariateDistributionType.Deterministic; - var xUnivAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); - if ( xUnivAttr != null) + var typeAttr = xElement.Attribute(nameof(UnivariateDistributionBase.Type)); + if (typeAttr != null) { - Enum.TryParse(xUnivAttr.Value, out type); + Enum.TryParse(typeAttr.Value, out type); } if (type == UnivariateDistributionType.PertPercentileZ) { bool parametersSolved = false; - var xParamSolvedAttr = xElement.Attribute("ParametersSolved"); - if (xParamSolvedAttr != null) + var paramsSolvedAttr = xElement.Attribute("ParametersSolved"); + if (paramsSolvedAttr != null) { - bool.TryParse(xParamSolvedAttr.Value, out parametersSolved); + bool.TryParse(paramsSolvedAttr.Value, out parametersSolved); } else { @@ -527,10 +527,10 @@ public static PertPercentileZ FromXElement(XElement xElement) var vals = new double[dist.NumberOfParameters]; for (int i = 0; i < dist.NumberOfParameters; i++) { - var xAttr = xElement.Attribute(names[i]); - if (xAttr != null) + var nameAttr = xElement.Attribute(names[i]); + if (nameAttr != null) { - double.TryParse(xAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out vals[i]); + double.TryParse(nameAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out vals[i]); } } dist.SetParameters(vals); @@ -539,10 +539,10 @@ public static PertPercentileZ FromXElement(XElement xElement) } var beta = new GeneralizedBeta(); - var xBetaAttr = xElement.Attribute("BetaParameters"); - if (xBetaAttr != null) + var betaParamsAttr = xElement.Attribute("BetaParameters"); + if (betaParamsAttr != null) { - var vals = xBetaAttr.Value.Split('|'); + var vals = betaParamsAttr.Value.Split('|'); var parameters = new List(); for (int i = 0; i < vals.Length; i++) { @@ -552,10 +552,10 @@ public static PertPercentileZ FromXElement(XElement xElement) beta.SetParameters(parameters); } double _5th = 0, _50th = 0, _95th = 0; - var xParamAttr = xElement.Attribute("Parameters"); - if (xParamAttr != null) + var paramsAttr = xElement.Attribute("Parameters"); + if (paramsAttr != null) { - var vals = xParamAttr.Value.Split('|'); + var vals = paramsAttr.Value.Split('|'); double.TryParse(vals[0], NumberStyles.Any, CultureInfo.InvariantCulture, out _5th); double.TryParse(vals[1], NumberStyles.Any, CultureInfo.InvariantCulture, out _50th); double.TryParse(vals[2], NumberStyles.Any, CultureInfo.InvariantCulture, out _95th); @@ -574,7 +574,7 @@ public static PertPercentileZ FromXElement(XElement xElement) } else { - return null!; + return null; } } diff --git a/Numerics/Distributions/Univariate/Poisson.cs b/Numerics/Distributions/Univariate/Poisson.cs index 9e8d1821..6ead9394 100644 --- a/Numerics/Distributions/Univariate/Poisson.cs +++ b/Numerics/Distributions/Univariate/Poisson.cs @@ -209,7 +209,7 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0]) || parameters[0] <= 0.0d) { diff --git a/Numerics/Distributions/Univariate/Rayleigh.cs b/Numerics/Distributions/Univariate/Rayleigh.cs index 8c74e25e..a007d2b6 100644 --- a/Numerics/Distributions/Univariate/Rayleigh.cs +++ b/Numerics/Distributions/Univariate/Rayleigh.cs @@ -210,7 +210,7 @@ public void Estimate(IList sample, ParameterEstimationMethod estimationM { if (estimationMethod == ParameterEstimationMethod.MethodOfMoments) { - SetParameters(Statistics.StandardDeviation(sample)); + SetParameters(Statistics.Mean(sample) / Math.Sqrt(Math.PI / 2.0d)); } else if (estimationMethod == ParameterEstimationMethod.MaximumLikelihood) { @@ -259,7 +259,7 @@ public override void SetParameters(IList parameters) /// /// The scale parameter σ (sigma). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double scale, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double scale, bool throwException) { if (double.IsNaN(scale) || double.IsInfinity(scale) || scale <= 0.0d) { @@ -271,7 +271,7 @@ public ArgumentOutOfRangeException ValidateParameters(double scale, bool throwEx } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], throwException); } diff --git a/Numerics/Distributions/Univariate/StudentT.cs b/Numerics/Distributions/Univariate/StudentT.cs index 599babb5..7f5fae35 100644 --- a/Numerics/Distributions/Univariate/StudentT.cs +++ b/Numerics/Distributions/Univariate/StudentT.cs @@ -108,7 +108,7 @@ public StudentT(double location, double scale, double degreesOfFreedom) private double _mu; private double _sigma; - private int _degreesOfFreedom; + private double _degreesOfFreedom; /// /// Gets and sets the location parameter µ (Mu). @@ -139,7 +139,7 @@ public double Sigma /// /// Gets and sets the degrees of freedom ν (nu) of the distribution. /// - public int DegreesOfFreedom + public double DegreesOfFreedom { get { return _degreesOfFreedom; } set @@ -325,7 +325,7 @@ public void SetParameters(double mu, double sigma, double v) { Mu = mu; Sigma = sigma; - DegreesOfFreedom = (int)v; + DegreesOfFreedom = v; } /// @@ -341,7 +341,7 @@ public override void SetParameters(IList parameters) /// The scale parameter σ (sigma). /// The degrees of freedom ν (nu). /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double location, double scale, double v, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double location, double scale, double v, bool throwException) { if (double.IsNaN(location) || double.IsInfinity(location)) { @@ -365,7 +365,7 @@ public ArgumentOutOfRangeException ValidateParameters(double location, double sc } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } diff --git a/Numerics/Distributions/Univariate/Triangular.cs b/Numerics/Distributions/Univariate/Triangular.cs index e16e098b..b5c26988 100644 --- a/Numerics/Distributions/Univariate/Triangular.cs +++ b/Numerics/Distributions/Univariate/Triangular.cs @@ -319,7 +319,7 @@ public override void SetParameters(IList parameters) /// The mode, or most likely, value of the distribution. /// The maximum possible value of the distribution. /// Determines whether to throw an exception or not. - private ArgumentOutOfRangeException ValidateParameters(double min, double mode, double max, bool throwException) + private ArgumentOutOfRangeException? ValidateParameters(double min, double mode, double max, bool throwException) { if (double.IsNaN(min) || double.IsInfinity(min) || double.IsNaN(max) || double.IsInfinity(max) || min > max) @@ -338,7 +338,7 @@ private ArgumentOutOfRangeException ValidateParameters(double min, double mode, } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], throwException); } @@ -452,15 +452,10 @@ public override double InverseCDF(double probability) { return Min + Math.Sqrt(probability * (Max - Min) * (MostLikely - Min)); } - else if (probability >= (MostLikely - Min) / (Max - Min)) + else { return Max - Math.Sqrt((1.0d - probability) * (Max - Min) * (Max - MostLikely)); } - else if (Max - Min == 0d) - { - return MostLikely; - } - return double.NaN; } /// diff --git a/Numerics/Distributions/Univariate/TruncatedDistribution.cs b/Numerics/Distributions/Univariate/TruncatedDistribution.cs index 6fb26359..368e7339 100644 --- a/Numerics/Distributions/Univariate/TruncatedDistribution.cs +++ b/Numerics/Distributions/Univariate/TruncatedDistribution.cs @@ -271,22 +271,22 @@ public override void SetParameters(IList parameters) } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { - if (_baseDist != null!) _baseDist.ValidateParameters(parameters.ToArray().Subset(0, parameters.Count - 2), throwException); + if (_baseDist is not null) _baseDist.ValidateParameters(parameters.ToArray().Subset(0, parameters.Count - 2), throwException); if (double.IsNaN(Min) || double.IsNaN(Max) || double.IsInfinity(Min) || double.IsInfinity(Max) || Min >= Max) { if (throwException) throw new ArgumentOutOfRangeException(nameof(Min), "The min must be less than the max."); return new ArgumentOutOfRangeException(nameof(Min), "The min must be less than the max."); } - if (_Fmin == _Fmax) + if (Math.Abs(_Fmin - _Fmax) < 1e-15) { if (throwException) throw new ArgumentOutOfRangeException(nameof(Min), "Truncation interval has zero probability mass."); return new ArgumentOutOfRangeException(nameof(Min), "Truncation interval has zero probability mass."); } - return null!; + return null; } /// diff --git a/Numerics/Distributions/Univariate/TruncatedNormal.cs b/Numerics/Distributions/Univariate/TruncatedNormal.cs index 465352bd..7ba57eaa 100644 --- a/Numerics/Distributions/Univariate/TruncatedNormal.cs +++ b/Numerics/Distributions/Univariate/TruncatedNormal.cs @@ -366,9 +366,9 @@ public override void SetParameters(IList parameters) /// The minimum possible value of the distribution. /// The maximum possible value of the distribution. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double mean, double standardDeviation, double min, double max, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double mean, double standardDeviation, double min, double max, bool throwException) { - if (double.IsNaN(mean) || double.IsInfinity(Mean)) + if (double.IsNaN(mean) || double.IsInfinity(mean)) { if (throwException) throw new ArgumentOutOfRangeException(nameof(Mu), "Mean must be a number."); @@ -390,7 +390,7 @@ public ArgumentOutOfRangeException ValidateParameters(double mean, double standa } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], parameters[2], parameters[3], throwException); } diff --git a/Numerics/Distributions/Univariate/Uncertainty Analysis/BootstrapAnalysis.cs b/Numerics/Distributions/Univariate/Uncertainty Analysis/BootstrapAnalysis.cs index 28c2b6b5..033a5ed0 100644 --- a/Numerics/Distributions/Univariate/Uncertainty Analysis/BootstrapAnalysis.cs +++ b/Numerics/Distributions/Univariate/Uncertainty Analysis/BootstrapAnalysis.cs @@ -173,6 +173,7 @@ public IUnivariateDistribution[] Distributions(ParameterSet[] parameterSets) // On fail, set to null if (failed == true) bootDistributions[idx] = null!; + }); return bootDistributions; } @@ -181,7 +182,7 @@ public IUnivariateDistribution[] Distributions(ParameterSet[] parameterSets) /// /// Bootstrap an array of distribution parameters. /// - public double[,] Parameters(IUnivariateDistribution[] distributions = null!) + public double[,] Parameters(IUnivariateDistribution[]? distributions = null) { var bootDistributions = distributions != null ? distributions : Distributions(); var bootParameters = new double[bootDistributions.Count(), Distribution.NumberOfParameters]; @@ -206,7 +207,7 @@ public IUnivariateDistribution[] Distributions(ParameterSet[] parameterSets) /// /// Bootstrap an array of distribution parameter sets. /// - public ParameterSet[] ParameterSets(IUnivariateDistribution[] distributions = null!) + public ParameterSet[] ParameterSets(IUnivariateDistribution[]? distributions = null) { var bootDistributions = distributions != null ? distributions : Distributions(); var bootParameters = new ParameterSet[bootDistributions.Count()]; @@ -293,7 +294,7 @@ public ParameterSet[] ParameterSets(IUnivariateDistribution[] distributions = nu /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. /// Optional. Pass in an array of bootstrapped distributions. Default = null. /// Optional. Determines whether to record parameter sets. Default = true. - public UncertaintyAnalysisResults Estimate(IList probabilities, double alpha = 0.1, IUnivariateDistribution[] distributions = null!, bool recordParameterSets = true) + public UncertaintyAnalysisResults Estimate(IList probabilities, double alpha = 0.1, IUnivariateDistribution[]? distributions = null, bool recordParameterSets = true) { var results = new UncertaintyAnalysisResults(); results.ParentDistribution = (UnivariateDistributionBase)Distribution; @@ -344,7 +345,7 @@ public UncertaintyAnalysisResults Estimate(IList probabilities, double a /// List quantile values. /// List of non-exceedance probabilities. /// Optional. Pass in an array of bootstrapped distributions. Default = null. - public double[] ExpectedProbabilities(IList quantiles, IList probabilities, IUnivariateDistribution[] distributions = null!) + public double[] ExpectedProbabilities(IList quantiles, IList probabilities, IUnivariateDistribution[]? distributions = null) { var quants = quantiles.ToArray(); var probs = probabilities.ToArray(); @@ -396,7 +397,7 @@ public double[] ExpectedProbabilities(IList quantiles, IList pro /// /// List quantile values. /// Optional. Pass in an array of bootstrapped distributions. Default = null. - public double[] ExpectedProbabilities(IList quantiles, IUnivariateDistribution[] distributions = null!) + public double[] ExpectedProbabilities(IList quantiles, IUnivariateDistribution[]? distributions = null) { var quants = quantiles.ToArray(); Array.Sort(quants); @@ -427,15 +428,18 @@ public double[] ExpectedProbabilities(IList quantiles, IUnivariateDistri public double[] ComputeMinMaxQuantiles(double minProbability, double maxProbability, IUnivariateDistribution[] distributions) { var output = new double[] { double.MaxValue, double.MinValue }; + object lockObject = new object(); Parallel.For(0, distributions.Count(), j => { if (distributions[j] != null) { var minX = distributions[j].InverseCDF(minProbability); - if (minX < output[0]) output[0] = minX; - var maxX = distributions[j].InverseCDF(maxProbability); - if (maxX > output[1]) output[1] = maxX; + lock (lockObject) + { + if (minX < output[0]) output[0] = minX; + if (maxX > output[1]) output[1] = maxX; + } } }); return output; @@ -447,7 +451,7 @@ public double[] ComputeMinMaxQuantiles(double minProbability, double maxProbabil /// List of non-exceedance probabilities. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. /// Optional. Pass in an array of bootstrapped distributions. Default = null. - public double[,] PercentileQuantileCI(IList probabilities, double alpha = 0.1, IUnivariateDistribution[] distributions = null!) + public double[,] PercentileQuantileCI(IList probabilities, double alpha = 0.1, IUnivariateDistribution[]? distributions = null) { var CIs = new double[] { alpha / 2d, 1d - alpha / 2d }; var Output = new double[probabilities.Count, 2]; @@ -457,8 +461,19 @@ public double[] ComputeMinMaxQuantiles(double minProbability, double maxProbabil var XValues = new double[bootDistributions.Count()]; Parallel.For(0, bootDistributions.Count(), idx => { XValues[idx] = bootDistributions[idx] != null ? bootDistributions[idx].InverseCDF(probabilities[i]) : double.NaN; }); - // sort X values - var validValues = XValues.Where(x => !double.IsNaN(x)).ToArray(); + // Filter valid values and sort + int validCount = 0; + for (int k = 0; k < XValues.Length; k++) + { + if (!double.IsNaN(XValues[k])) validCount++; + } + var validValues = new double[validCount]; + int writeIdx = 0; + for (int k = 0; k < XValues.Length; k++) + { + if (!double.IsNaN(XValues[k])) + validValues[writeIdx++] = XValues[k]; + } Array.Sort(validValues); // Record percentiles for CIs @@ -474,7 +489,7 @@ public double[] ComputeMinMaxQuantiles(double minProbability, double maxProbabil /// List of non-exceedance probabilities. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. /// Optional. Pass in an array of bootstrapped distributions. Default = null. - public double[,] BiasCorrectedQuantileCI(IList probabilities, double alpha = 0.1, IUnivariateDistribution[] distributions = null!) + public double[,] BiasCorrectedQuantileCI(IList probabilities, double alpha = 0.1, IUnivariateDistribution[]? distributions = null) { // Create list of original X values given probability values var populationXValues = new double[probabilities.Count]; @@ -498,8 +513,19 @@ public double[] ComputeMinMaxQuantiles(double minProbability, double maxProbabil // get proportion P0 = P0 / (bootDistributions.Count() + 1); - // sort X values - var validValues = XValues.Where(x => !double.IsNaN(x)).ToArray(); + // Filter valid values and sort + int validCount = 0; + for (int k = 0; k < XValues.Length; k++) + { + if (!double.IsNaN(XValues[k])) validCount++; + } + var validValues = new double[validCount]; + int writeIdx = 0; + for (int k = 0; k < XValues.Length; k++) + { + if (!double.IsNaN(XValues[k])) + validValues[writeIdx++] = XValues[k]; + } Array.Sort(validValues); // Record percentiles for CIs @@ -520,7 +546,7 @@ public double[] ComputeMinMaxQuantiles(double minProbability, double maxProbabil /// List of non-exceedance probabilities. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. /// Optional. Pass in an array of bootstrapped distributions. Default = null. - public double[,] NormalQuantileCI(IList probabilities, double alpha = 0.1, IUnivariateDistribution[] distributions = null!) + public double[,] NormalQuantileCI(IList probabilities, double alpha = 0.1, IUnivariateDistribution[]? distributions = null) { // Create list of original X values given probability values @@ -537,8 +563,21 @@ public double[] ComputeMinMaxQuantiles(double minProbability, double maxProbabil var XValues = new double[bootDistributions.Count()]; Parallel.For(0, bootDistributions.Count(), idx => { XValues[idx] = bootDistributions[idx] != null ? Math.Pow(bootDistributions[idx].InverseCDF(probabilities[i]), 1d / 3d) : double.NaN; }); + // Filter valid values + int validCount = 0; + for (int k = 0; k < XValues.Length; k++) + { + if (!double.IsNaN(XValues[k])) validCount++; + } + var validValues = new double[validCount]; + int writeIdx = 0; + for (int k = 0; k < XValues.Length; k++) + { + if (!double.IsNaN(XValues[k])) + validValues[writeIdx++] = XValues[k]; + } + // Get Standard error - var validValues = XValues.Where(x => !double.IsNaN(x)).ToArray(); double SE = Statistics.StandardDeviation(validValues); // Record percentiles for CIs @@ -592,8 +631,19 @@ public double[] ComputeMinMaxQuantiles(double minProbability, double maxProbabil // get proportion P0 = (P0 + 1) / (Replications + 1); - // sort X values - var validValues = XValues.Where(x => !double.IsNaN(x)).ToArray(); + // Filter valid values and sort + int validCount = 0; + for (int k = 0; k < XValues.Length; k++) + { + if (!double.IsNaN(XValues[k])) validCount++; + } + var validValues = new double[validCount]; + int writeIdx = 0; + for (int k = 0; k < XValues.Length; k++) + { + if (!double.IsNaN(XValues[k])) + validValues[writeIdx++] = XValues[k]; + } Array.Sort(validValues); // Record percentiles for CIs @@ -726,8 +776,25 @@ private double[] AccelerationConstants(IList sampleData, IList p for (int i = 0; i < probabilities.Count; i++) { - var XValues = xValues.GetColumn(i).Where(x => !double.IsNaN(x)).ToArray(); - var TValues = studentT.GetColumn(i).Where(x => !double.IsNaN(x)).ToArray(); + var rawX = xValues.GetColumn(i); + var rawT = studentT.GetColumn(i); + int validCount = 0; + for (int k = 0; k < rawX.Length; k++) + { + if (!double.IsNaN(rawX[k])) validCount++; + } + var XValues = new double[validCount]; + var TValues = new double[validCount]; + int writeIdx = 0; + for (int k = 0; k < rawX.Length; k++) + { + if (!double.IsNaN(rawX[k])) + { + XValues[writeIdx] = rawX[k]; + TValues[writeIdx] = rawT[k]; + writeIdx++; + } + } // Get Standard error double SE = Statistics.StandardDeviation(XValues); diff --git a/Numerics/Distributions/Univariate/Uncertainty Analysis/UncertaintyAnalysisResults.cs b/Numerics/Distributions/Univariate/Uncertainty Analysis/UncertaintyAnalysisResults.cs index 29ff0a46..b476e2ac 100644 --- a/Numerics/Distributions/Univariate/Uncertainty Analysis/UncertaintyAnalysisResults.cs +++ b/Numerics/Distributions/Univariate/Uncertainty Analysis/UncertaintyAnalysisResults.cs @@ -81,7 +81,7 @@ public UncertaintyAnalysisResults(UnivariateDistributionBase parentDistribution, double maxProbability = 1 - 1e-9, bool recordParameterSets = false) { - if (parentDistribution == null!) + if (parentDistribution is null) throw new ArgumentNullException(nameof(parentDistribution)); if (sampledDistributions == null || sampledDistributions.Length == 0) throw new ArgumentException("Sampled distributions cannot be null or empty.", nameof(sampledDistributions)); @@ -106,27 +106,27 @@ public UncertaintyAnalysisResults(UnivariateDistributionBase parentDistribution, /// /// The parent probability distribution. /// - public UnivariateDistributionBase ParentDistribution { get; set; } = null!; + public UnivariateDistributionBase? ParentDistribution { get; set; } /// /// The array of parameter sets. /// - public ParameterSet[] ParameterSets { get; set; } = null!; + public ParameterSet[]? ParameterSets { get; set; } /// - /// The confidence intervals. + /// The confidence intervals. /// - public double[,] ConfidenceIntervals { get; set; } = null!; + public double[,]? ConfidenceIntervals { get; set; } /// - /// The mode (or computed) curve from the parent distribution. + /// The mode (or computed) curve from the parent distribution. /// - public double[] ModeCurve { get; set; } = null!; + public double[]? ModeCurve { get; set; } /// - /// The mean (or predictive) curve. + /// The mean (or predictive) curve. /// - public double[] MeanCurve { get; set; } = null!; + public double[]? MeanCurve { get; set; } /// /// Gets or sets the Akaike information criteria (AIC) of the fit. @@ -176,7 +176,7 @@ public static byte[] ToByteArray(UncertaintyAnalysisResults results) /// Returns the class from a byte array. /// /// Byte array. - public static UncertaintyAnalysisResults FromByteArray(byte[] bytes) + public static UncertaintyAnalysisResults? FromByteArray(byte[] bytes) { try { @@ -189,22 +189,21 @@ public static UncertaintyAnalysisResults FromByteArray(byte[] bytes) options.Converters.Add(new Double2DArrayConverter()); options.Converters.Add(new String2DArrayConverter()); options.Converters.Add(new UnivariateDistributionConverter()); - var result = JsonSerializer.Deserialize(bytes, options); - return result ?? FromByteArrayLegacy(bytes); + return JsonSerializer.Deserialize(bytes, options); } catch (Exception) { - // An error can occur because we're trying to deserialize a blob written with binary formatter, - //as a blob of json bytes. If that happens, fall back to the old. + // An error can occur because we're trying to deserialize a blob written with binary formatter, + //as a blob of json bytes. If that happens, fall back to the old. return FromByteArrayLegacy(bytes); - } + } } /// - /// Returns the class from a byte array. + /// Returns the class from a byte array using the legacy BinaryFormatter. /// /// Byte array. - private static UncertaintyAnalysisResults FromByteArrayLegacy(byte[] bytes) + private static UncertaintyAnalysisResults? FromByteArrayLegacy(byte[] bytes) { try { @@ -227,7 +226,7 @@ private static UncertaintyAnalysisResults FromByteArrayLegacy(byte[] bytes) // If there is an error, just catch it and force the user to rerun the // uncertainty analysis. } - return null!; + return null; } /// @@ -236,7 +235,7 @@ private static UncertaintyAnalysisResults FromByteArrayLegacy(byte[] bytes) public XElement ToXElement() { var result = new XElement(nameof(UncertaintyAnalysisResults)); - if (ParentDistribution != null!) result.Add(ParentDistribution.ToXElement()); + if (ParentDistribution is not null) result.Add(ParentDistribution.ToXElement()); result.SetAttributeValue(nameof(AIC), AIC.ToString("G17", CultureInfo.InvariantCulture)); result.SetAttributeValue(nameof(BIC), BIC.ToString("G17", CultureInfo.InvariantCulture)); result.SetAttributeValue(nameof(DIC), DIC.ToString("G17", CultureInfo.InvariantCulture)); @@ -267,64 +266,53 @@ public XElement ToXElement() /// XElement to deserialize. public static UncertaintyAnalysisResults FromXElement(XElement xElement) { - var ua = new UncertaintyAnalysisResults(); + var ua = new UncertaintyAnalysisResults(); // Parent distribution var distElement = xElement.Element("Distribution"); if (distElement != null) - { - var parentDist = UnivariateDistributionFactory.CreateDistribution(distElement); - if (parentDist is not null) - { - ua.ParentDistribution = parentDist; - } - else - { - throw new InvalidDataException("Unable to deserialize parent distribution from XElement."); - } - } - + ua.ParentDistribution = UnivariateDistributionFactory.CreateDistribution(distElement); // AIC - var aicElement = xElement.Attribute(nameof(AIC)); - if (aicElement != null) + var aicAttr = xElement.Attribute(nameof(AIC)); + if (aicAttr != null) { - double.TryParse(aicElement.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var aic); + double.TryParse(aicAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var aic); ua.AIC = aic; } // BIC - var bicElement = xElement.Attribute(nameof(BIC)); - if (bicElement != null) + var bicAttr = xElement.Attribute(nameof(BIC)); + if (bicAttr != null) { - double.TryParse(bicElement.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var bic); + double.TryParse(bicAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var bic); ua.BIC = bic; } // DIC - var dicElement = xElement.Attribute(nameof(DIC)); - if (dicElement != null) + var dicAttr = xElement.Attribute(nameof(DIC)); + if (dicAttr != null) { - double.TryParse(dicElement.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var dic); + double.TryParse(dicAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var dic); ua.DIC = dic; } // RMSE - var rmseElement = xElement.Attribute(nameof(RMSE)); - if (rmseElement != null) + var rmseAttr = xElement.Attribute(nameof(RMSE)); + if (rmseAttr != null) { - double.TryParse(rmseElement.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var rmse); + double.TryParse(rmseAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var rmse); ua.RMSE = rmse; } // ERL - var erlElement = xElement.Attribute(nameof(ERL)); - if (erlElement != null) + var erlAttr = xElement.Attribute(nameof(ERL)); + if (erlAttr != null) { - double.TryParse(erlElement.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var erl); + double.TryParse(erlAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var erl); ua.ERL = erl; } // Mode Curve - var modeAttr = xElement.Attribute(nameof(ua.ModeCurve)); - if (modeAttr != null) + var modeCurveAttr = xElement.Attribute(nameof(ua.ModeCurve)); + if (modeCurveAttr != null) { - var vals = modeAttr.Value.Split('|'); + var vals = modeCurveAttr.Value.Split('|'); if (vals.Length > 0) { ua.ModeCurve = new double[vals.Length]; @@ -335,10 +323,10 @@ public static UncertaintyAnalysisResults FromXElement(XElement xElement) } } // Mean Curve - var meanAttr = xElement.Attribute(nameof(ua.MeanCurve)); - if (meanAttr != null) + var meanCurveAttr = xElement.Attribute(nameof(ua.MeanCurve)); + if (meanCurveAttr != null) { - var vals = meanAttr.Value.Split('|'); + var vals = meanCurveAttr.Value.Split('|'); if (vals.Length > 0) { ua.MeanCurve = new double[vals.Length]; @@ -376,7 +364,7 @@ public static UncertaintyAnalysisResults FromXElement(XElement xElement) /// Array of non-exceedance probabilities. public void ProcessModeCurve(UnivariateDistributionBase parentDistribution, double[] probabilities) { - if (parentDistribution == null!) + if (parentDistribution is null) throw new ArgumentNullException(nameof(parentDistribution)); if (probabilities == null || probabilities.Length == 0) throw new ArgumentException("Probabilities cannot be null or empty.", nameof(probabilities)); @@ -463,7 +451,7 @@ public void ProcessMeanCurve(UnivariateDistributionBase[] sampledDistributions, Parallel.For(0, B, j => { - if (sampledDistributions[j] != null!) + if (sampledDistributions[j] is not null) { var innerMin = sampledDistributions[j].InverseCDF(minProbability); var innerMax = sampledDistributions[j].InverseCDF(maxProbability); @@ -499,7 +487,7 @@ public void ProcessMeanCurve(UnivariateDistributionBase[] sampledDistributions, double total = 0d; Parallel.For(0, B, () => 0d, (j, loop, sum) => { - if (sampledDistributions[j] != null!) + if (sampledDistributions[j] is not null) { sum += sampledDistributions[j].CDF(quantiles[i]); } @@ -511,8 +499,8 @@ public void ProcessMeanCurve(UnivariateDistributionBase[] sampledDistributions, // Build monotonic interpolation points var yVals = new List { quantiles[0] }; var xVals = new List { expected[0] }; - double minY = quantiles[0]; - double maxY = quantiles[0]; + double minY = double.MaxValue; + double maxY = double.MinValue; for (int i = 1; i < bins; i++) { @@ -551,7 +539,7 @@ public void ProcessParameterSets(UnivariateDistributionBase[] sampledDistributio Parallel.For(0, B, idx => { - if (sampledDistributions[idx] != null!) + if (sampledDistributions[idx] is not null) { ParameterSets[idx] = new ParameterSet(sampledDistributions[idx].GetParameters, double.NaN); } diff --git a/Numerics/Distributions/Univariate/Uniform.cs b/Numerics/Distributions/Univariate/Uniform.cs index 9497e43f..b5448bad 100644 --- a/Numerics/Distributions/Univariate/Uniform.cs +++ b/Numerics/Distributions/Univariate/Uniform.cs @@ -242,7 +242,7 @@ public override void SetParameters(IList parameters) /// The min of the distribution. /// The max of the distribution. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double min, double max, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double min, double max, bool throwException) { if (double.IsNaN(min) || double.IsInfinity(min) || double.IsNaN(max) || double.IsInfinity(max) || min > max) @@ -255,7 +255,7 @@ public ArgumentOutOfRangeException ValidateParameters(double min, double max, bo } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } diff --git a/Numerics/Distributions/Univariate/UniformDiscrete.cs b/Numerics/Distributions/Univariate/UniformDiscrete.cs index ab603bec..7421a384 100644 --- a/Numerics/Distributions/Univariate/UniformDiscrete.cs +++ b/Numerics/Distributions/Univariate/UniformDiscrete.cs @@ -185,7 +185,11 @@ public override double Mode /// public override double StandardDeviation { - get { return Math.Sqrt((Max - Min) * (Max - Min) / 12.0d); } + get + { + double n = Max - Min + 1; + return Math.Sqrt((n * n - 1.0d) / 12.0d); + } } /// @@ -250,7 +254,7 @@ public override void SetParameters(IList parameters) /// The min of the distribution. /// The max of the distribution. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double min, double max, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double min, double max, bool throwException) { if (double.IsNaN(min) || double.IsInfinity(min) || double.IsNaN(max) || double.IsInfinity(max) || min > max) @@ -263,7 +267,7 @@ public ArgumentOutOfRangeException ValidateParameters(double min, double max, bo } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } diff --git a/Numerics/Distributions/Univariate/VonMises.cs b/Numerics/Distributions/Univariate/VonMises.cs new file mode 100644 index 00000000..02dc3247 --- /dev/null +++ b/Numerics/Distributions/Univariate/VonMises.cs @@ -0,0 +1,485 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Collections.Generic; +using Numerics.Mathematics.Integration; +using Numerics.Mathematics.Optimization; +using Numerics.Mathematics.RootFinding; +using Numerics.Mathematics.SpecialFunctions; + +namespace Numerics.Distributions +{ + + /// + /// The von Mises distribution for circular data. + /// + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// Description: + /// + /// + /// The von Mises distribution (also known as the circular normal distribution) is a continuous + /// probability distribution on the circle. It is the circular analogue of the normal distribution + /// and is commonly used to model directional data such as flood seasonality (timing of annual + /// maximum flows), wind directions, and other angular measurements. + /// + /// + /// References: + /// + /// + /// + /// + /// Mardia, K.V. and Jupp, P.E. (2000). "Directional Statistics." John Wiley and Sons. + /// + /// + /// Best, D.J. and Fisher, N.I. (1979). "Efficient simulation of the von Mises distribution." + /// Applied Statistics, 28(2), 152-157. + /// + /// + /// + /// + /// + /// + /// + [Serializable] + public sealed class VonMises : UnivariateDistributionBase, IEstimation, IMaximumLikelihoodEstimation, IBootstrappable + { + + /// + /// Constructs a von Mises distribution with μ = 0 and κ = 1. + /// + public VonMises() + { + SetParameters(0d, 1d); + } + + /// + /// Constructs a von Mises distribution with given μ and κ. + /// + /// The mean direction μ (mu), in radians. Must be in [-π, π]. + /// The concentration parameter κ (kappa). Must be ≥ 0. + public VonMises(double mu, double kappa) + { + SetParameters(mu, kappa); + } + + private double _mu; // mean direction + private double _kappa; // concentration + + /// + /// Gets and sets the mean direction parameter μ (mu), in radians. + /// + public double Mu + { + get { return _mu; } + set + { + _parametersValid = ValidateParameters([value, Kappa], false) is null; + _mu = value; + } + } + + /// + /// Gets and sets the concentration parameter κ (kappa). + /// + public double Kappa + { + get { return _kappa; } + set + { + _parametersValid = ValidateParameters([Mu, value], false) is null; + _kappa = value; + } + } + + /// + public override int NumberOfParameters + { + get { return 2; } + } + + /// + public override UnivariateDistributionType Type + { + get { return UnivariateDistributionType.VonMises; } + } + + /// + public override string DisplayName + { + get { return "Von Mises"; } + } + + /// + public override string ShortDisplayName + { + get { return "VM"; } + } + + /// + public override string[,] ParametersToString + { + get + { + var parmString = new string[2, 2]; + parmString[0, 0] = "Mean Direction (μ)"; + parmString[1, 0] = "Concentration (κ)"; + parmString[0, 1] = Mu.ToString(); + parmString[1, 1] = Kappa.ToString(); + return parmString; + } + } + + /// + public override string[] ParameterNamesShortForm + { + get { return ["μ", "κ"]; } + } + + /// + public override string[] GetParameterPropertyNames + { + get { return [nameof(Mu), nameof(Kappa)]; } + } + + /// + public override double[] GetParameters + { + get { return [Mu, Kappa]; } + } + + /// + public override double Mean + { + get { return Mu; } + } + + /// + public override double Median + { + get { return Mu; } + } + + /// + public override double Mode + { + get { return Mu; } + } + + /// + /// Gets the circular standard deviation, defined as √(1 - I₁(κ)/I₀(κ)). + /// + /// + /// The circular variance is defined as V = 1 - A(κ), where A(κ) = I₁(κ)/I₀(κ) + /// is the mean resultant length. The standard deviation is √V. + /// For κ = 0 (uniform), V = 1. As κ → ∞, V → 0. + /// Note: Variance is computed as StandardDeviation² by the base class. + /// + public override double StandardDeviation + { + get { return Math.Sqrt(1d - Bessel.I1(_kappa) / Bessel.I0(_kappa)); } + } + + /// + public override double Skewness + { + get { return 0d; } + } + + /// + public override double Kurtosis + { + get + { + // Circular kurtosis is not directly comparable to linear kurtosis + return double.NaN; + } + } + + /// + public override double Minimum + { + get { return -Math.PI; } + } + + /// + public override double Maximum + { + get { return Math.PI; } + } + + /// + public override double[] MinimumOfParameters + { + get { return [-Math.PI, 0d]; } + } + + /// + public override double[] MaximumOfParameters + { + get { return [Math.PI, double.PositiveInfinity]; } + } + + /// + public void Estimate(IList sample, ParameterEstimationMethod estimationMethod) + { + if (estimationMethod == ParameterEstimationMethod.MaximumLikelihood) + { + SetParameters(MLE(sample)); + } + else + { + throw new NotImplementedException(); + } + } + + /// + public IUnivariateDistribution Bootstrap(ParameterEstimationMethod estimationMethod, int sampleSize, int seed = -1) + { + var newDistribution = new VonMises(Mu, Kappa); + var sample = newDistribution.GenerateRandomValues(sampleSize, seed); + newDistribution.Estimate(sample, estimationMethod); + if (newDistribution.ParametersValid == false) + throw new Exception("Bootstrapped distribution parameters are invalid."); + return newDistribution; + } + + /// + /// Set the distribution parameters. + /// + /// The mean direction μ (mu), in radians. + /// The concentration parameter κ (kappa). + public void SetParameters(double mu, double kappa) + { + _parametersValid = ValidateParameters(new[] { mu, kappa }, false) is null; + _mu = mu; + _kappa = kappa; + } + + /// + public override void SetParameters(IList parameters) + { + SetParameters(parameters[0], parameters[1]); + } + + /// + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) + { + if (double.IsNaN(parameters[0]) || double.IsInfinity(parameters[0])) + { + if (throwException) + throw new ArgumentOutOfRangeException(nameof(Mu), "The mean direction parameter μ (mu) must be a number."); + return new ArgumentOutOfRangeException(nameof(Mu), "The mean direction parameter μ (mu) must be a number."); + } + if (parameters[0] < -Math.PI || parameters[0] > Math.PI) + { + if (throwException) + throw new ArgumentOutOfRangeException(nameof(Mu), "The mean direction parameter μ (mu) must be in [-π, π]."); + return new ArgumentOutOfRangeException(nameof(Mu), "The mean direction parameter μ (mu) must be in [-π, π]."); + } + if (double.IsNaN(parameters[1]) || double.IsInfinity(parameters[1]) || parameters[1] < 0d) + { + if (throwException) + throw new ArgumentOutOfRangeException(nameof(Kappa), "The concentration parameter κ (kappa) must be non-negative."); + return new ArgumentOutOfRangeException(nameof(Kappa), "The concentration parameter κ (kappa) must be non-negative."); + } + return null; + } + + /// + public Tuple GetParameterConstraints(IList sample) + { + var initialVals = MLE(sample); + var lowerVals = new double[] { -Math.PI, 0d }; + var upperVals = new double[] { Math.PI, Math.Max(initialVals[1] * 10d, 100d) }; + return new Tuple(initialVals, lowerVals, upperVals); + } + + /// + public double[] MLE(IList sample) + { + // Compute mean direction + double sumSin = 0d, sumCos = 0d; + for (int i = 0; i < sample.Count; i++) + { + sumSin += Math.Sin(sample[i]); + sumCos += Math.Cos(sample[i]); + } + double mu = Math.Atan2(sumSin, sumCos); + + // Compute mean resultant length R_bar + double rBar = Math.Sqrt(sumSin * sumSin + sumCos * sumCos) / sample.Count; + + // Solve A(kappa) = R_bar for kappa, where A(kappa) = I1(kappa)/I0(kappa) + // Use the approximation from Mardia & Jupp (2000) as initial estimate + double kappa; + if (rBar < 0.53) + { + kappa = 2d * rBar + rBar * rBar * rBar + 5d / 6d * Math.Pow(rBar, 5); + } + else if (rBar < 0.85) + { + kappa = -0.4 + 1.39 * rBar + 0.43 / (1d - rBar); + } + else + { + kappa = 1d / (rBar * rBar * rBar - 4d * rBar * rBar + 3d * rBar); + } + + // Refine with Newton-Raphson iterations: A(kappa) = I1/I0, A'(kappa) = 1 - A(kappa)^2 - A(kappa)/kappa + if (rBar > 0 && rBar < 1) + { + for (int i = 0; i < 20; i++) + { + double a = Bessel.I1(kappa) / Bessel.I0(kappa); + double aPrime = 1d - a * a - a / kappa; + if (Math.Abs(aPrime) < 1e-30) break; + double delta = (a - rBar) / aPrime; + kappa -= delta; + if (kappa < 0) kappa = Tools.DoubleMachineEpsilon; + if (Math.Abs(delta) < 1e-12) break; + } + } + else if (rBar >= 1) + { + kappa = double.MaxValue; + } + else + { + kappa = 0d; + } + + return [mu, kappa]; + } + + /// + public override double PDF(double x) + { + if (_parametersValid == false) + ValidateParameters([Mu, Kappa], true); + if (x < -Math.PI || x > Math.PI) + return 0d; + return Math.Exp(_kappa * Math.Cos(x - _mu)) / (2d * Math.PI * Bessel.I0(_kappa)); + } + + /// + public override double CDF(double x) + { + if (_parametersValid == false) + ValidateParameters([Mu, Kappa], true); + if (x <= -Math.PI) return 0d; + if (x >= Math.PI) return 1d; + + // Numerical integration from -π to x + var integrator = new AdaptiveGaussKronrod((t) => Math.Exp(_kappa * Math.Cos(t - _mu)), -Math.PI, x); + integrator.Integrate(); + return integrator.Result / (2d * Math.PI * Bessel.I0(_kappa)); + } + + /// + public override double InverseCDF(double probability) + { + if (probability < 0d || probability > 1d) + throw new ArgumentOutOfRangeException("probability", "Probability must be between 0 and 1."); + if (probability == 0d) return Minimum; + if (probability == 1d) return Maximum; + if (_parametersValid == false) + ValidateParameters([Mu, Kappa], true); + + // Use Brent's method to solve CDF(x) = probability + return Brent.Solve((x) => CDF(x) - probability, -Math.PI, Math.PI); + } + + /// + public override UnivariateDistributionBase Clone() + { + return new VonMises(Mu, Kappa); + } + + /// + /// Generates random values from the von Mises distribution using Best's algorithm. + /// + /// + /// Best, D.J. and Fisher, N.I. (1979). "Efficient simulation of the von Mises distribution." + /// + public override double[] GenerateRandomValues(int sampleSize, int seed = -1) + { + if (_parametersValid == false) + ValidateParameters([Mu, Kappa], true); + + var rng = seed < 0 ? new Random() : new Random(seed); + var values = new double[sampleSize]; + + if (_kappa < 1e-10) + { + // For kappa ≈ 0, the distribution is approximately uniform on [-π, π] + for (int i = 0; i < sampleSize; i++) + values[i] = -Math.PI + 2d * Math.PI * rng.NextDouble(); + return values; + } + + // Best's algorithm + double tau = 1d + Math.Sqrt(1d + 4d * _kappa * _kappa); + double rho = (tau - Math.Sqrt(2d * tau)) / (2d * _kappa); + double r = (1d + rho * rho) / (2d * rho); + + for (int i = 0; i < sampleSize; i++) + { + double f, c; + while (true) + { + double u1 = rng.NextDouble(); + double z = Math.Cos(Math.PI * u1); + f = (1d + r * z) / (r + z); + c = _kappa * (r - f); + + double u2 = rng.NextDouble(); + if (c * (2d - c) > u2 || Math.Log(c / u2) + 1d >= c) + break; + } + + double u3 = rng.NextDouble(); + double theta = (u3 > 0.5 ? 1d : -1d) * Math.Acos(f) + _mu; + + // Wrap to [-π, π] + theta = ((theta + Math.PI) % (2d * Math.PI) + 2d * Math.PI) % (2d * Math.PI) - Math.PI; + values[i] = theta; + } + + return values; + } + + } +} diff --git a/Numerics/Distributions/Univariate/Weibull.cs b/Numerics/Distributions/Univariate/Weibull.cs index 1f8a5bc4..0e399fc7 100644 --- a/Numerics/Distributions/Univariate/Weibull.cs +++ b/Numerics/Distributions/Univariate/Weibull.cs @@ -290,7 +290,7 @@ public override void SetParameters(IList parameters) /// The scale parameter λ (lambda). Range: λ > 0. /// The shape parameter κ (kappa). Range: k > 0. /// Determines whether to throw an exception or not. - public ArgumentOutOfRangeException ValidateParameters(double scale, double shape, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(double scale, double shape, bool throwException) { if (double.IsNaN(scale) || double.IsInfinity(scale) || scale <= 0.0d) { @@ -308,7 +308,7 @@ public ArgumentOutOfRangeException ValidateParameters(double scale, double shape } /// - public override ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public override ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { return ValidateParameters(parameters[0], parameters[1], throwException); } diff --git a/Numerics/Functions/IUnivariateFunction.cs b/Numerics/Functions/IUnivariateFunction.cs index 5ecae6b2..3af50b90 100644 --- a/Numerics/Functions/IUnivariateFunction.cs +++ b/Numerics/Functions/IUnivariateFunction.cs @@ -97,7 +97,7 @@ public interface IUnivariateFunction /// Array of parameters. /// Boolean indicating whether to throw the exception or not. /// Nothing if the parameters are valid and the exception if invalid parameters were found. - ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException); + ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException); /// /// Returns the function evaluated at a point x. If function is uncertain, the function is computed at the set confidence level. diff --git a/Numerics/Functions/LinearFunction.cs b/Numerics/Functions/LinearFunction.cs index 9d4dee0f..f66b2ff1 100644 --- a/Numerics/Functions/LinearFunction.cs +++ b/Numerics/Functions/LinearFunction.cs @@ -162,10 +162,11 @@ public void SetParameters(IList parameters) _alpha = parameters[0]; _beta = parameters[1]; _sigma = parameters[2]; + _normal.SetParameters(0, _sigma); } /// - public ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (IsDeterministic == false && parameters[2] <= 0) { @@ -205,6 +206,8 @@ public double InverseFunction(double y) // Validate parameters if (_parametersValid == false) ValidateParameters(new[] { Alpha, Beta, Sigma }, true); + if (Math.Abs(Beta) < double.Epsilon) + throw new InvalidOperationException("Cannot compute inverse function when Beta is zero."); double x = 0; if (IsDeterministic == true || ConfidenceLevel < 0 || ConfidenceLevel > 1) diff --git a/Numerics/Functions/Link Functions/ComplementaryLogLogLink.cs b/Numerics/Functions/Link Functions/ComplementaryLogLogLink.cs new file mode 100644 index 00000000..c74deec9 --- /dev/null +++ b/Numerics/Functions/Link Functions/ComplementaryLogLogLink.cs @@ -0,0 +1,98 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Xml.Linq; + +namespace Numerics.Functions +{ + /// + /// Complementary log-log link function mapping the unit interval (0, 1) to the unconstrained real line. + /// + /// + /// + /// Domain: x ∈ (0, 1). h(x) = log(−log(1 − x)), h⁻¹(η) = 1 − exp(−exp(η)), + /// h′(x) = 1 / ((1 − x) · (−log(1 − x))). + /// + /// + /// The complementary log-log link is used for asymmetric binary response models, + /// particularly when the probability of the event is small (rare events). It arises + /// naturally from extreme value (Gumbel) latent variable models. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public class ComplementaryLogLogLink : ILinkFunction + { + /// + /// Smallest admissible x to avoid log(0) blow-ups near the boundary. + /// + private const double MinX = 1e-12; + + /// + /// Largest admissible x to avoid log(0) blow-ups near the boundary. + /// + private const double MaxX = 1.0 - 1e-12; + + /// + /// Thrown when is not in (0, 1). + public double Link(double x) + { + if (x <= 0.0 || x >= 1.0) + throw new ArgumentOutOfRangeException(nameof(x), "ComplementaryLogLogLink requires x in (0, 1)."); + x = Math.Max(MinX, Math.Min(MaxX, x)); + return Math.Log(-Math.Log(1.0 - x)); + } + + /// + public double InverseLink(double eta) + { + return 1.0 - Math.Exp(-Math.Exp(eta)); + } + + /// + /// Thrown when is not in (0, 1). + public double DLink(double x) + { + if (x <= 0.0 || x >= 1.0) + throw new ArgumentOutOfRangeException(nameof(x), "ComplementaryLogLogLink derivative requires x in (0, 1)."); + x = Math.Max(MinX, Math.Min(MaxX, x)); + // h'(x) = 1 / ((1 - x) * (-log(1 - x))) + double oneMinusX = 1.0 - x; + double negLogOneMinusX = -Math.Log(oneMinusX); + return 1.0 / Math.Max(oneMinusX * negLogOneMinusX, 1e-16); + } + + /// + public XElement ToXElement() => new XElement(nameof(ComplementaryLogLogLink)); + } +} diff --git a/Numerics/Functions/Link Functions/ILinkFunction.cs b/Numerics/Functions/Link Functions/ILinkFunction.cs new file mode 100644 index 00000000..29689724 --- /dev/null +++ b/Numerics/Functions/Link Functions/ILinkFunction.cs @@ -0,0 +1,82 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System.Xml.Linq; + +namespace Numerics.Functions +{ + /// + /// Interface for link functions that transform parameters between real-space and link-space. + /// + /// + /// + /// A link function h provides a bijective, differentiable mapping from the parameter's + /// natural domain to an unconstrained link-space. This is used in generalized linear models (GLMs), + /// Bayesian MCMC estimation, and parametric bootstrap procedures. + /// + /// + /// Implementations must satisfy the round-trip identity: InverseLink(Link(x)) = x + /// for all x in the valid domain, and the derivative consistency: DLink(x) = dLink(x)/dx. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public interface ILinkFunction + { + /// + /// Evaluates the link function mapping real-space to link-space: η = h(x). + /// + /// The real-space value to transform. + /// The link-space value η. + double Link(double x); + + /// + /// Evaluates the inverse link function mapping link-space back to real-space: x = h⁻¹(η). + /// + /// The link-space value to transform. + /// The real-space value x. + double InverseLink(double eta); + + /// + /// Evaluates the derivative of the link function with respect to x: h′(x) = dη/dx. + /// + /// The real-space value at which to evaluate the derivative. + /// The derivative dη/dx. + double DLink(double x); + + /// + /// Serializes the link function to an . + /// + /// An XElement representing the link function and its parameters. + XElement ToXElement(); + } +} diff --git a/Numerics/Functions/Link Functions/IdentityLink.cs b/Numerics/Functions/Link Functions/IdentityLink.cs new file mode 100644 index 00000000..fefedcbb --- /dev/null +++ b/Numerics/Functions/Link Functions/IdentityLink.cs @@ -0,0 +1,65 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System.Xml.Linq; + +namespace Numerics.Functions +{ + /// + /// Identity link function: h(x) = x. No transformation is applied. + /// + /// + /// + /// The identity link maps parameters directly without transformation. + /// It is the canonical link for the Normal (Gaussian) GLM family. + /// + /// + /// Domain: all real numbers. h(x) = x, h⁻¹(η) = η, h′(x) = 1. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public class IdentityLink : ILinkFunction + { + /// + public double Link(double x) => x; + + /// + public double InverseLink(double eta) => eta; + + /// + public double DLink(double x) => 1.0; + + /// + public XElement ToXElement() => new XElement(nameof(IdentityLink)); + } +} diff --git a/Numerics/Functions/Link Functions/LinkController.cs b/Numerics/Functions/Link Functions/LinkController.cs new file mode 100644 index 00000000..51f7dbde --- /dev/null +++ b/Numerics/Functions/Link Functions/LinkController.cs @@ -0,0 +1,249 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Linq; +using System.Xml.Linq; +using Numerics.Mathematics.LinearAlgebra; + +namespace Numerics.Functions +{ + /// + /// Controller for managing independent link functions applied to each element of a parameter vector. + /// + /// + /// + /// Each parameter index can have its own link function, or null for no transformation (identity). + /// This enables variance stabilization, constraint enforcement, and improved MCMC mixing + /// by applying per-parameter transformations. + /// + /// + /// The Jacobian and log-determinant methods support the change-of-variables formula + /// for transformed-space probability calculations: p(φ) = p(θ) |∂θ/∂φ|. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public class LinkController + { + /// + /// The array of link functions, one per parameter index. Null entries indicate identity (no transformation). + /// + public ILinkFunction?[] Links { get; private set; } + + /// + /// Initializes a new instance with no link functions (all parameters untransformed). + /// + public LinkController() + { + Links = Array.Empty(); + } + + /// + /// Initializes a new instance of the class with the specified link functions. + /// + /// + /// An array of link functions, one per parameter index. Null entries indicate identity (no transformation). + /// If no arguments are provided, all parameters pass through untransformed. + /// + public LinkController(params ILinkFunction?[] links) + { + Links = links ?? Array.Empty(); + } + + /// + /// Initializes a new instance from a serialized . + /// + /// The XElement to deserialize from. + /// Thrown when is null. + /// + /// + /// The XElement is expected to contain child elements named "Link" with an "Index" attribute + /// and optional child elements representing the link function type. + /// Uses for deserialization, + /// which supports only standard Numerics link types. For BestFit-specific types (SESLink, etc.), + /// use the constructor with custom deserialization. + /// + /// + public LinkController(XElement xElement) + { + if (xElement == null) throw new ArgumentNullException(nameof(xElement)); + var slots = xElement.Elements("Link").ToList(); + Links = new ILinkFunction?[slots.Count]; + foreach (var slot in slots) + { + var indexAttr = slot.Attribute("Index"); + if (indexAttr == null) continue; + if (!int.TryParse(indexAttr.Value, out int index)) continue; + if (index < 0 || index >= Links.Length) continue; + var child = slot.Elements().FirstOrDefault(); + if (child != null) + Links[index] = LinkFunctionFactory.CreateFromXElement(child); + } + } + + /// + /// Gets the number of link functions registered. + /// + public int Count => Links.Length; + + /// + /// Gets the link function at the specified parameter index, or null if no link is assigned. + /// + /// The zero-based parameter index. + /// The link function at the index, or null if the index is out of range or no link is assigned. + public ILinkFunction? this[int index] => + index >= 0 && index < Links.Length ? Links[index] : null; + + /// + /// Applies the link functions element-wise: η[i] = h_i(x[i]). + /// + /// The parameter vector in real-space coordinates. + /// The transformed parameter vector in link-space. + public double[] Link(double[] x) + { + var eta = (double[])x.Clone(); + int n = Math.Min(x.Length, Links.Length); + for (int i = 0; i < n; i++) + { + if (Links[i] != null) + eta[i] = Links[i]!.Link(x[i]); + } + return eta; + } + + /// + /// Applies the inverse link functions element-wise: x[i] = h_i⁻¹(η[i]). + /// + /// The parameter vector in link-space coordinates. + /// The parameter vector in real-space. + public double[] InverseLink(double[] eta) + { + var x = (double[])eta.Clone(); + int n = Math.Min(eta.Length, Links.Length); + for (int i = 0; i < n; i++) + { + if (Links[i] != null) + x[i] = Links[i]!.InverseLink(eta[i]); + } + return x; + } + + /// + /// Computes the diagonal Jacobian matrix of the link transformation. + /// + /// The parameter vector in real-space coordinates. + /// A diagonal matrix with elements dη_i/dθ_i for each parameter. + /// + /// + /// The Jacobian is diagonal because each parameter has an independent link function. + /// Off-diagonal elements are zero. + /// + /// + public Matrix LinkJacobian(double[] x) + { + int p = x.Length; + var G = Matrix.Identity(p); + int n = Math.Min(p, Links.Length); + for (int i = 0; i < n; i++) + { + if (Links[i] != null) + G[i, i] = Links[i]!.DLink(x[i]); + } + return G; + } + + /// + /// Computes the log-determinant of the inverse Jacobian |∂θ/∂φ| for the change-of-variables formula. + /// + /// The parameter vector in link-space (transformed coordinates). + /// log|det(∂θ/∂φ)|, the log absolute determinant of the inverse Jacobian. + /// + /// + /// Used in transformed-space probability calculations: p(φ) = p(θ) |∂θ/∂φ|. + /// + /// + /// For diagonal link Jacobians: log|det| = −∑ log|dη_j/dθ_j|. + /// + /// + public double LogDetJacobian(double[] phi) + { + const double tiny = 1e-16; + double sum = 0.0; + int n = Math.Min(phi.Length, Links.Length); + for (int i = 0; i < n; i++) + { + if (Links[i] != null) + { + double theta_i = Links[i]!.InverseLink(phi[i]); + double dEta_dTheta = Links[i]!.DLink(theta_i); + // Use Math.Abs to handle both increasing and decreasing link functions. + sum -= Math.Log(Math.Max(Math.Abs(dEta_dTheta), tiny)); + } + } + return sum; + } + + /// + /// Serializes the link controller to an . + /// + /// An XElement containing the serialized link functions with their indices. + public XElement ToXElement() + { + var element = new XElement(nameof(LinkController)); + for (int i = 0; i < Links.Length; i++) + { + var slot = new XElement("Link"); + slot.SetAttributeValue("Index", i); + if (Links[i] != null) + slot.Add(Links[i]!.ToXElement()); + element.Add(slot); + } + return element; + } + + /// + /// Creates a for the standard 3-parameter (location, scale, shape) case. + /// + /// Optional link function for the location parameter. + /// Optional link function for the scale parameter. + /// Optional link function for the shape parameter. + /// A new configured for three parameters. + public static LinkController ForLocationScaleShape( + ILinkFunction? locationLink = null, + ILinkFunction? scaleLink = null, + ILinkFunction? shapeLink = null) + { + return new LinkController(locationLink, scaleLink, shapeLink); + } + } +} diff --git a/Numerics/Functions/Link Functions/LinkFunctionFactory.cs b/Numerics/Functions/Link Functions/LinkFunctionFactory.cs new file mode 100644 index 00000000..3ea38d5b --- /dev/null +++ b/Numerics/Functions/Link Functions/LinkFunctionFactory.cs @@ -0,0 +1,101 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Xml.Linq; + +namespace Numerics.Functions +{ + /// + /// Factory for creating instances from enum values + /// or from serialized representations. + /// + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public static class LinkFunctionFactory + { + /// + /// Creates an instance corresponding to the specified link function type. + /// + /// The link function type. + /// A new instance. + /// Thrown when is not a recognized link function type. + public static ILinkFunction Create(LinkFunctionType type) + { + switch (type) + { + case LinkFunctionType.Identity: + return new IdentityLink(); + case LinkFunctionType.Log: + return new LogLink(); + case LinkFunctionType.Logit: + return new LogitLink(); + case LinkFunctionType.Probit: + return new ProbitLink(); + case LinkFunctionType.ComplementaryLogLog: + return new ComplementaryLogLogLink(); + default: + throw new ArgumentOutOfRangeException(nameof(type), $"Unknown link function type: {type}."); + } + } + + /// + /// Creates an instance from a serialized . + /// The element name determines the link function type. + /// + /// The XElement representing the link function. The element name must match a known link function class name. + /// A new instance. + /// Thrown when is null. + /// Thrown when the element name does not correspond to a known link function type. + public static ILinkFunction CreateFromXElement(XElement xElement) + { + if (xElement == null) throw new ArgumentNullException(nameof(xElement)); + switch (xElement.Name.LocalName) + { + case nameof(IdentityLink): + return new IdentityLink(); + case nameof(LogLink): + return new LogLink(); + case nameof(LogitLink): + return new LogitLink(); + case nameof(ProbitLink): + return new ProbitLink(); + case nameof(ComplementaryLogLogLink): + return new ComplementaryLogLogLink(); + default: + throw new NotSupportedException($"Unknown link function type: '{xElement.Name.LocalName}'."); + } + } + } +} diff --git a/Numerics/Functions/Link Functions/LinkFunctionType.cs b/Numerics/Functions/Link Functions/LinkFunctionType.cs new file mode 100644 index 00000000..060da8f1 --- /dev/null +++ b/Numerics/Functions/Link Functions/LinkFunctionType.cs @@ -0,0 +1,74 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +namespace Numerics.Functions +{ + /// + /// Enumeration of standard link function types for generalized linear models. + /// + /// + /// + /// Each value corresponds to a canonical link for a specific GLM family. + /// Use to obtain + /// an instance from an enum value. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public enum LinkFunctionType + { + /// + /// Identity link: η = x. Canonical link for the Normal (Gaussian) family. + /// + Identity, + + /// + /// Log link: η = log(x). Canonical link for the Poisson and Exponential families. + /// + Log, + + /// + /// Logit link: η = log(x / (1 − x)). Canonical link for the Binomial family. + /// + Logit, + + /// + /// Probit link: η = Φ⁻¹(x). Alternative link for the Binomial family using the standard normal quantile function. + /// + Probit, + + /// + /// Complementary log-log link: η = log(−log(1 − x)). Used for asymmetric binary response models. + /// + ComplementaryLogLog + } +} diff --git a/Numerics/Functions/Link Functions/LogLink.cs b/Numerics/Functions/Link Functions/LogLink.cs new file mode 100644 index 00000000..a8f86e2b --- /dev/null +++ b/Numerics/Functions/Link Functions/LogLink.cs @@ -0,0 +1,88 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Xml.Linq; + +namespace Numerics.Functions +{ + /// + /// Log link function mapping positive reals to the unconstrained real line. + /// + /// + /// + /// Domain: x > 0. h(x) = log(x), h⁻¹(η) = exp(η), h′(x) = 1/x. + /// + /// + /// The log link is the canonical link for the Poisson and Exponential GLM families. + /// It is commonly used for scale parameters that must remain positive. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public class LogLink : ILinkFunction + { + /// + /// Smallest admissible x to avoid log(0) and 1/x blow-ups. + /// + private const double MinX = 1e-12; + + /// + /// Thrown when is less than or equal to zero. + public double Link(double x) + { + if (x <= 0.0) + throw new ArgumentOutOfRangeException(nameof(x), "LogLink requires x > 0."); + if (x < MinX) x = MinX; + return Math.Log(x); + } + + /// + public double InverseLink(double eta) + { + return Math.Exp(eta); + } + + /// + /// Thrown when is less than or equal to zero. + public double DLink(double x) + { + if (x <= 0.0) + throw new ArgumentOutOfRangeException(nameof(x), "LogLink derivative requires x > 0."); + if (x < MinX) x = MinX; + return 1.0 / x; + } + + /// + public XElement ToXElement() => new XElement(nameof(LogLink)); + } +} diff --git a/Numerics/Functions/Link Functions/LogitLink.cs b/Numerics/Functions/Link Functions/LogitLink.cs new file mode 100644 index 00000000..08c286b0 --- /dev/null +++ b/Numerics/Functions/Link Functions/LogitLink.cs @@ -0,0 +1,108 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Xml.Linq; + +namespace Numerics.Functions +{ + /// + /// Logit link function mapping the unit interval (0, 1) to the unconstrained real line. + /// + /// + /// + /// Domain: x ∈ (0, 1). h(x) = log(x / (1 − x)), h⁻¹(η) = 1 / (1 + exp(−η)), h′(x) = 1 / (x(1 − x)). + /// + /// + /// The logit link is the canonical link for the Binomial GLM family. + /// It is used for probability parameters, mixing proportions, and any parameter + /// naturally bounded to the unit interval. + /// + /// + /// The inverse link (sigmoid/logistic function) uses numerically stable formulation + /// to avoid overflow for large |η|. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public class LogitLink : ILinkFunction + { + /// + /// Smallest admissible x to avoid log(0) blow-ups near the boundary. + /// + private const double MinX = 1e-12; + + /// + /// Largest admissible x to avoid log(0) blow-ups near the boundary. + /// + private const double MaxX = 1.0 - 1e-12; + + /// + /// Thrown when is not in (0, 1). + public double Link(double x) + { + if (x <= 0.0 || x >= 1.0) + throw new ArgumentOutOfRangeException(nameof(x), "LogitLink requires x in (0, 1)."); + x = Math.Max(MinX, Math.Min(MaxX, x)); + return Math.Log(x / (1.0 - x)); + } + + /// + public double InverseLink(double eta) + { + // Numerically stable sigmoid to avoid overflow for large |eta|. + if (eta >= 0) + { + double e = Math.Exp(-eta); + return 1.0 / (1.0 + e); + } + else + { + double e = Math.Exp(eta); + return e / (1.0 + e); + } + } + + /// + /// Thrown when is not in (0, 1). + public double DLink(double x) + { + if (x <= 0.0 || x >= 1.0) + throw new ArgumentOutOfRangeException(nameof(x), "LogitLink derivative requires x in (0, 1)."); + x = Math.Max(MinX, Math.Min(MaxX, x)); + return 1.0 / (x * (1.0 - x)); + } + + /// + public XElement ToXElement() => new XElement(nameof(LogitLink)); + } +} diff --git a/Numerics/Functions/Link Functions/ProbitLink.cs b/Numerics/Functions/Link Functions/ProbitLink.cs new file mode 100644 index 00000000..208c8f52 --- /dev/null +++ b/Numerics/Functions/Link Functions/ProbitLink.cs @@ -0,0 +1,98 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Xml.Linq; +using Numerics.Distributions; + +namespace Numerics.Functions +{ + /// + /// Probit link function mapping the unit interval (0, 1) to the unconstrained real line + /// using the standard normal quantile function. + /// + /// + /// + /// Domain: x ∈ (0, 1). h(x) = Φ⁻¹(x), h⁻¹(η) = Φ(η), h′(x) = 1 / φ(Φ⁻¹(x)), + /// where Φ is the standard normal CDF and φ is the standard normal PDF. + /// + /// + /// The probit link is an alternative link for the Binomial GLM family. + /// It assumes an underlying normally distributed latent variable. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + public class ProbitLink : ILinkFunction + { + /// + /// Smallest admissible x to avoid quantile blow-ups near the boundary. + /// + private const double MinX = 1e-12; + + /// + /// Largest admissible x to avoid quantile blow-ups near the boundary. + /// + private const double MaxX = 1.0 - 1e-12; + + /// + /// Thrown when is not in (0, 1). + public double Link(double x) + { + if (x <= 0.0 || x >= 1.0) + throw new ArgumentOutOfRangeException(nameof(x), "ProbitLink requires x in (0, 1)."); + x = Math.Max(MinX, Math.Min(MaxX, x)); + return Normal.StandardZ(x); + } + + /// + public double InverseLink(double eta) + { + return Normal.StandardCDF(eta); + } + + /// + /// Thrown when is not in (0, 1). + public double DLink(double x) + { + if (x <= 0.0 || x >= 1.0) + throw new ArgumentOutOfRangeException(nameof(x), "ProbitLink derivative requires x in (0, 1)."); + x = Math.Max(MinX, Math.Min(MaxX, x)); + double z = Normal.StandardZ(x); + double pdf = Normal.StandardPDF(z); + return 1.0 / Math.Max(pdf, 1e-16); + } + + /// + public XElement ToXElement() => new XElement(nameof(ProbitLink)); + } +} diff --git a/Numerics/Functions/PowerFunction.cs b/Numerics/Functions/PowerFunction.cs index 900ac1e7..1386eac9 100644 --- a/Numerics/Functions/PowerFunction.cs +++ b/Numerics/Functions/PowerFunction.cs @@ -156,17 +156,17 @@ public double Sigma public double Minimum { get { return Xi; } - set { return; } + set { throw new NotSupportedException("Minimum is derived from Xi and cannot be set directly."); } } /// public double Maximum { get; set; } = double.MaxValue; /// - public double[] MinimumOfParameters => new double[] { 0, -10, 0 }; + public double[] MinimumOfParameters => new double[] { 0, -10, 0, 0 }; /// - public double[] MaximumOfParameters => new double[] { double.MaxValue, 10, double.MaxValue }; + public double[] MaximumOfParameters => new double[] { double.MaxValue, 10, double.MaxValue, double.MaxValue }; /// public bool IsDeterministic { get; set; } @@ -189,10 +189,11 @@ public void SetParameters(IList parameters) _beta = parameters[1]; _xi = parameters[2]; _sigma = parameters[3]; + _normal.SetParameters(0, _sigma); } /// - public ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { if (IsDeterministic == false && parameters[3] <= 0) { diff --git a/Numerics/Functions/TabularFunction.cs b/Numerics/Functions/TabularFunction.cs index 8e374875..b1f33a1a 100644 --- a/Numerics/Functions/TabularFunction.cs +++ b/Numerics/Functions/TabularFunction.cs @@ -150,7 +150,7 @@ public void SetParameters(IList parameters) } /// - public ArgumentOutOfRangeException ValidateParameters(IList parameters, bool throwException) + public ArgumentOutOfRangeException? ValidateParameters(IList parameters, bool throwException) { var errors = PairedData.GetErrors(); if (errors.Count > 0) diff --git a/Numerics/Machine Learning/Supervised/DecisionTree.cs b/Numerics/Machine Learning/Supervised/DecisionTree.cs index bab82c7d..0333f807 100644 --- a/Numerics/Machine Learning/Supervised/DecisionTree.cs +++ b/Numerics/Machine Learning/Supervised/DecisionTree.cs @@ -428,15 +428,15 @@ private double TraverseTree(double[] x, DecisionNode node) if (node.IsLeafNode == true) return node.Value; if (x[node.FeatureIndex] <= node.Threshold) - return TraverseTree(x, node.Left); - return TraverseTree(x, node.Right); + return node.Left != null ? TraverseTree(x, node.Left) : node.Value; + return node.Right != null ? TraverseTree(x, node.Right) : node.Value; } /// /// Returns the prediction from the Decision Tree. /// /// The 1D array of predictors. - public double[] Predict(double[] X) + public double[]? Predict(double[] X) { return Predict(new Matrix(X)); } @@ -445,7 +445,7 @@ public double[] Predict(double[] X) /// Returns the prediction from the Decision Tree. /// /// The 2D array of predictors. - public double[] Predict(double[,] X) + public double[]? Predict(double[,] X) { return Predict(new Matrix(X)); } @@ -454,7 +454,7 @@ public double[] Predict(double[,] X) /// Returns the prediction from the Decision Tree. /// /// The matrix of predictors. - public double[] Predict(Matrix X) + public double[]? Predict(Matrix X) { if (!IsTrained || X.NumberOfColumns != Dimensions) return null!; var result = new double[X.NumberOfRows]; diff --git a/Numerics/Machine Learning/Supervised/GeneralizedLinearModel.cs b/Numerics/Machine Learning/Supervised/GeneralizedLinearModel.cs index e36ff85c..c5358187 100644 --- a/Numerics/Machine Learning/Supervised/GeneralizedLinearModel.cs +++ b/Numerics/Machine Learning/Supervised/GeneralizedLinearModel.cs @@ -1,4 +1,4 @@ -/* +/* * NOTICE: * The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about * the results, or appropriateness of outputs, obtained from Numerics. @@ -30,6 +30,7 @@ using Numerics.Data.Statistics; using Numerics.Distributions; +using Numerics.Functions; using Numerics.Mathematics.LinearAlgebra; using Numerics.Mathematics.Optimization; using Numerics.Mathematics.SpecialFunctions; @@ -38,7 +39,7 @@ namespace Numerics.MachineLearning { /// - /// A class for performing generalized linear regression. + /// A class for performing generalized linear regression. /// /// /// @@ -51,13 +52,26 @@ public class GeneralizedLinearModel #region Construction /// - /// Constructs a generalized linear model. + /// Constructs a generalized linear model. /// /// The matrix of predictor values. /// The response vector. /// Determines if an intercept should be estimate. Default = true. /// The link function type. Default = identity. - public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, LinkFunctionType linkType = LinkFunctionType.Identity) + public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, LinkFunctionType linkType = LinkFunctionType.Identity) + : this(x, y, LinkFunctionFactory.Create(linkType), hasIntercept, linkType) + { + } + + /// + /// Constructs a generalized linear model with a custom link function. + /// + /// The matrix of predictor values. + /// The response vector. + /// The link function instance. + /// Determines if an intercept should be estimated. Default = true. + /// The link function type for optimizer initialization. Default = identity. + public GeneralizedLinearModel(Matrix x, Vector y, ILinkFunction linkFunction, bool hasIntercept = true, LinkFunctionType linkType = LinkFunctionType.Identity) { if (y.Length != x.NumberOfRows) throw new ArgumentException("X and Y must have the same number of rows."); if (y.Length <= 2) throw new ArithmeticException("There must be at least three data points."); @@ -91,7 +105,8 @@ public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, Link } LinkType = linkType; - (_inverseLink, _inverseLinkDerivative, _logLikelihoodTerm, _logLikelihoodGradient) = GetLinkFunctions(linkType); + LinkFunction = linkFunction; + _logLikelihoodTerm = GetFamilyLogLikelihood(linkType); SetOptimizer(); } @@ -101,7 +116,7 @@ public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, Link #region Members /// - /// Determines if the linear model has an intercept. + /// Determines if the linear model has an intercept. /// public bool HasIntercept { get; private set; } @@ -111,44 +126,44 @@ public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, Link public Vector Y { get; private set; } /// - /// The matrix of predictor values. + /// The matrix of predictor values. /// public Matrix X { get; private set; } /// /// The list of estimated parameter values. /// - public double[] Parameters { get; private set; } = Array.Empty(); + public double[] Parameters { get; private set; } = null!; /// - /// The list of the estimated parameter names. + /// The list of the estimated parameter names. /// - public List ParameterNames { get; private set; } + public List ParameterNames { get; private set; } = null!; /// - /// The list of the estimated parameter standard errors. + /// The list of the estimated parameter standard errors. /// - public double[] ParameterStandardErrors { get; private set; } = Array.Empty(); + public double[] ParameterStandardErrors { get; private set; } = null!; /// /// The list of the estimated parameter z-scores. /// - public double[] ParameterZScores { get; private set; } = Array.Empty(); + public double[] ParameterZScores { get; private set; } = null!; /// /// The list of the estimated parameter p-values. /// - public double[] ParameterPValues { get; private set; } = Array.Empty(); + public double[] ParameterPValues { get; private set; } = null!; /// - /// The estimate parameter covariance matrix. + /// The estimate parameter covariance matrix. /// - public Matrix Covariance { get; private set; } = new Matrix(1, 1); + public Matrix Covariance { get; private set; } = null!; /// - /// The residuals of the fitted linear model. + /// The residuals of the fitted linear model. /// - public double[] Residuals { get; private set; } = Array.Empty(); + public double[] Residuals { get; private set; } = null!; /// /// The model standard error. @@ -156,7 +171,7 @@ public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, Link public double StandardError { get; private set; } /// - /// The data sample size. + /// The data sample size. /// public int SampleSize => Y.Length; @@ -166,7 +181,7 @@ public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, Link public int DegreesOfFreedom { get; private set; } /// - /// the Akaike Information Criterion (AIC). + /// the Akaike Information Criterion (AIC). /// public double AIC { get; private set; } @@ -176,7 +191,7 @@ public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, Link public double AICc { get; private set; } /// - /// The Bayesian information criterion (BIC). + /// The Bayesian information criterion (BIC). /// public double BIC { get; private set; } @@ -191,31 +206,43 @@ public GeneralizedLinearModel(Matrix x, Vector y, bool hasIntercept = true, Link public Optimizer Optimizer { get; private set; } = null!; /// - /// Gets the link function type. + /// Gets the link function type. /// public LinkFunctionType LinkType { get; private set; } /// - /// Enumeration of link function types. + /// Gets the link function instance used by this model. /// - public enum LinkFunctionType - { - Identity, - Log, - Logit, - Probit, - ComplementaryLogLog - } + public ILinkFunction LinkFunction { get; private set; } - private Func _inverseLink; - private Func _inverseLinkDerivative; + /// + /// Per-observation log-likelihood contribution as a function of (mu, y). + /// This is distribution-family-specific (Normal, Poisson, Binomial). + /// private Func<(double mu, double y), double> _logLikelihoodTerm; - private Func<(double eta, double y), double> _logLikelihoodGradient; #endregion #region Methods + /// + /// Prepares the design matrix for prediction by adding an intercept column if needed. + /// If the matrix already has the expected number of columns (e.g. internal X), it passes through. + /// If it has one fewer column and HasIntercept is true, the intercept column is added. + /// + /// The matrix of predictor values. + private Matrix PrepareDesignMatrix(Matrix x) + { + int expected = Parameters.Length; + if (x.NumberOfColumns == expected) + return x; + if (HasIntercept && x.NumberOfColumns == expected - 1) + return AddInterceptColumn(x); + throw new ArgumentException( + $"Expected {expected} columns{(HasIntercept ? $" (or {expected - 1} without intercept)" : "")}, but got {x.NumberOfColumns}.", + nameof(x)); + } + /// /// Helper method to add an intercept column to the covariate matrix. /// @@ -233,106 +260,56 @@ private static Matrix AddInterceptColumn(Matrix x) } /// - /// Returns the necessary link functions. + /// Returns the distribution-family-specific per-observation log-likelihood function. /// - /// The link function type. - private static (Func InverseLink, - Func InverseLinkDerivative, - Func<(double mu, double y), double> LogLikelihoodTerm, - Func<(double eta, double y), double> LogLikelihoodGradientTerm) - GetLinkFunctions(LinkFunctionType type) + /// The link function type, which determines the assumed distribution family. + /// A function that computes the per-observation log-likelihood given (mu, y). + private static Func<(double mu, double y), double> GetFamilyLogLikelihood(LinkFunctionType type) { switch (type) { - // ------------------ Identity Link (Normal) ------------------ + // Normal family (Identity link) case LinkFunctionType.Identity: - return ( - eta => eta, - eta => 1.0, - (pair) => - { - double resid = pair.y - pair.mu; - return -0.5 * resid * resid; - }, - (pair) => (pair.y - pair.eta) * 1.0 // d/dη of SSE - ); - // ------------------ Log Link (Poisson) ------------------ + return (pair) => + { + double resid = pair.y - pair.mu; + return -0.5 * resid * resid; + }; + + // Poisson family (Log link) case LinkFunctionType.Log: - return ( - eta => Math.Exp(eta), - eta => Math.Exp(eta), - (pair) => - { - double mu = pair.mu; - return pair.y * Math.Log(mu) - mu; - }, - (pair) => - { - double mu = Math.Exp(pair.eta); - return (pair.y - mu) * mu; // mu * dμ/dη - } - ); - // ------------------ Logit Link (Binomial) ------------------ + return (pair) => + { + double mu = pair.mu; + return pair.y * Math.Log(mu) - mu; + }; + + // Binomial family (Logit, Probit, or Complementary Log-Log link) case LinkFunctionType.Logit: - return ( - eta => 1.0 / (1.0 + Math.Exp(-eta)), - eta => - { - double ex = Math.Exp(-eta); - return ex / Math.Pow(1 + ex, 2); - }, - (pair) => - { - double mu = pair.mu; - return pair.y * Math.Log(mu) + (1 - pair.y) * Math.Log(1 - mu); - }, - (pair) => - { - double mu = 1.0 / (1.0 + Math.Exp(-pair.eta)); - double dMu_dEta = mu * (1 - mu); - return (pair.y - mu) * dMu_dEta; - } - ); - // ------------------ Probit Link (Binomial) ------------------ case LinkFunctionType.Probit: - return ( - eta => Normal.StandardCDF(eta), - eta => Normal.StandardPDF(eta), - (pair) => - { - double mu = pair.mu; - return pair.y * Math.Log(mu) + (1 - pair.y) * Math.Log(1 - mu); - }, - (pair) => - { - double mu = Normal.StandardCDF(pair.eta); - double dMu = Normal.StandardPDF(pair.eta); - return (pair.y - mu) * dMu; - } - ); - // ------------------ Complementary Log-Log Link (Binomial) ------------------ case LinkFunctionType.ComplementaryLogLog: - return ( - eta => 1.0 - Math.Exp(-Math.Exp(eta)), - eta => Math.Exp(eta - Math.Exp(eta)), - (pair) => - { - double mu = pair.mu; - return pair.y * Math.Log(mu) + (1 - pair.y) * Math.Log(1 - mu); - }, - (pair) => - { - double mu = 1.0 - Math.Exp(-Math.Exp(pair.eta)); - double dMu = Math.Exp(pair.eta - Math.Exp(pair.eta)); - return (pair.y - mu) * dMu; - } - ); + return (pair) => + { + double mu = pair.mu; + return pair.y * Math.Log(mu) + (1 - pair.y) * Math.Log(1 - mu); + }; default: throw new ArgumentOutOfRangeException(nameof(type)); } } + /// + /// Computes the inverse link derivative dμ/dη at the given link-space value. + /// + /// The link-space value. + /// The derivative dμ/dη = 1 / h′(μ). + private double InverseLinkDerivative(double eta) + { + double mu = LinkFunction.InverseLink(eta); + double dEta_dMu = LinkFunction.DLink(mu); + return 1.0 / dEta_dMu; + } /// /// Set up the local optimizer. @@ -349,7 +326,8 @@ double logLikelihood(double[] beta) double logLH = 0.0; for (int i = 0; i < n; i++) { - logLH += _logLikelihoodTerm((_inverseLink(X.GetRow(i).DotProduct(beta)), Y[i])); + double mu = LinkFunction.InverseLink(X.GetRow(i).DotProduct(beta)); + logLH += _logLikelihoodTerm((mu, Y[i])); } if (double.IsNaN(logLH) || double.IsInfinity(logLH)) return double.MaxValue; return -logLH; @@ -363,7 +341,10 @@ double[] gradient(double[] beta) { Vector xi = new Vector(X.Row(i)); double eta = xi.Array.DotProduct(beta); - g -= xi * _logLikelihoodGradient((eta, Y[i])); + double mu = LinkFunction.InverseLink(eta); + double dMu_dEta = InverseLinkDerivative(eta); + double gradTerm = (Y[i] - mu) * dMu_dEta; + g -= xi * gradTerm; } return g.Array; } @@ -388,8 +369,8 @@ double[] gradient(double[] beta) if (HasIntercept) { initial[0] = (min + max) / 2.0; - lower[0] = initial[0] / 100; - upper[0] = initial[0] * 100; + lower[0] = Math.Min(initial[0] / 100, initial[0] * 100); + upper[0] = Math.Max(initial[0] / 100, initial[0] * 100); } } else if (LinkType == LinkFunctionType.Log) @@ -408,8 +389,8 @@ double[] gradient(double[] beta) if (HasIntercept) { initial[0] = (min + max) / 2.0; - lower[0] = initial[0] / 100; - upper[0] = initial[0] * 100; + lower[0] = Math.Min(initial[0] / 100, initial[0] * 100); + upper[0] = Math.Max(initial[0] / 100, initial[0] * 100); } } else if (LinkType == LinkFunctionType.Logit) @@ -544,7 +525,7 @@ private void ComputeDiagnostics() { var xi = X.GetRow(i); double eta = xi.DotProduct(Parameters); - double dMu = _inverseLinkDerivative(eta); + double dMu = InverseLinkDerivative(eta); for (int j = 0; j < p; j++) J[i, j] = dMu * xi[j]; } @@ -575,7 +556,7 @@ private void ComputeDiagnostics() } /// - /// Provides a standard summary output table in a list of strings. + /// Provides a standard summary output table in a list of strings. /// public List Summary() { @@ -599,7 +580,7 @@ public List Summary() } text.Add("---"); - text.Add("Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1"); + text.Add("Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1"); text.Add(""); text.Add($"Residual standard error: {StandardError:N4} on {DegreesOfFreedom} degrees of freedom"); text.Add($"AIC: {AIC:N4} AICc: {AICc:N4} BIC: {BIC:N4}"); @@ -616,46 +597,33 @@ public List Summary() } /// - /// Returns the mean prediction. + /// Returns the mean prediction. /// /// The matrix of predictors. public double[] Predict(Matrix x) { - int n = x.NumberOfRows; + var xp = PrepareDesignMatrix(x); + int n = xp.NumberOfRows; var result = new double[n]; for (int i = 0; i < n; i++) - result[i] = _inverseLink(x.GetRow(i).DotProduct(Parameters.ToArray())); + result[i] = LinkFunction.InverseLink(xp.GetRow(i).DotProduct(Parameters.ToArray())); return result; } /// - /// Returns the prediction with confidence intervals in a 2D array with columns: lower, mean, upper. + /// Returns the prediction with confidence intervals in a 2D array with columns: lower, mean, upper. /// /// The matrix of predictors. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. public double[,] Predict(Matrix x, double alpha = 0.1) { + var xp = PrepareDesignMatrix(x); var z = Normal.StandardZ(1 - alpha / 2); - var result = new double[x.NumberOfRows, 3]; - for (int i = 0; i < x.NumberOfRows; i++) + var result = new double[xp.NumberOfRows, 3]; + for (int i = 0; i < xp.NumberOfRows; i++) { - double[] xi; - if (HasIntercept) - { - int p = x.NumberOfColumns; - xi = new double[x.NumberOfColumns + 1]; - xi[0] = 1; - for (int j = 1; j < p; j++) - { - xi[i] = x[i, j]; - } - } - else - { - xi = x.GetRow(i); - } - - double mu = _inverseLink(xi.DotProduct(Parameters.ToArray())); + var xi = xp.GetRow(i); + double mu = LinkFunction.InverseLink(xi.DotProduct(Parameters.ToArray())); double se = Math.Sqrt(xi.DotProduct(Covariance.Multiply(new Vector(xi)).Array)); result[i, 0] = mu - z * se; result[i, 1] = mu; diff --git a/Numerics/Machine Learning/Supervised/KNearestNeighbors.cs b/Numerics/Machine Learning/Supervised/KNearestNeighbors.cs index f8c7bdfd..b2fc6474 100644 --- a/Numerics/Machine Learning/Supervised/KNearestNeighbors.cs +++ b/Numerics/Machine Learning/Supervised/KNearestNeighbors.cs @@ -115,7 +115,7 @@ public KNearestNeighbors(Matrix x, Vector y, int k) #region Members /// - /// The number of clusters. + /// The number of nearest neighbors. /// public int K { get; private set; } @@ -147,7 +147,7 @@ public KNearestNeighbors(Matrix x, Vector y, int k) /// Returns the indexes of the k-Nearest Neighbors. /// /// The 1D array of predictors. - public int[] GetNeighbors(double[] X) + public int[]? GetNeighbors(double[] X) { return kNN(this.X, this.Y, new Matrix(X)); } @@ -156,7 +156,7 @@ public int[] GetNeighbors(double[] X) /// Returns the indexes of the k-Nearest Neighbors. /// /// The 2D array of predictors. - public int[] GetNeighbors(double[,] X) + public int[]? GetNeighbors(double[,] X) { return kNN(this.X, this.Y, new Matrix(X)); } @@ -165,7 +165,7 @@ public int[] GetNeighbors(double[,] X) /// Returns the indexes of the k-Nearest Neighbors. /// /// The matrix of predictors. - public int[] GetNeighbors(Matrix X) + public int[]? GetNeighbors(Matrix X) { return kNN(this.X, this.Y, X); } @@ -174,7 +174,7 @@ public int[] GetNeighbors(Matrix X) /// Returns the prediction from k-Nearest neighbors. /// /// The 1D array of predictors. - public double[] Predict(double[] X) + public double[]? Predict(double[] X) { return kNNPredict(this.X, this.Y, new Matrix(X)); } @@ -183,16 +183,16 @@ public double[] Predict(double[] X) /// Returns the prediction from k-Nearest neighbors. /// /// The 2D array of predictors. - public double[] Predict(double[,] X) + public double[]? Predict(double[,] X) { - return kNNPredict(this.X, this.Y, new Matrix(X)); + return kNNPredict(this.X, this.Y, new Matrix(X)); } /// /// Returns the prediction from k-Nearest neighbors. /// /// The matrix of predictors. - public double[] Predict(Matrix X) + public double[]? Predict(Matrix X) { return kNNPredict(this.X, this.Y, X); } @@ -202,7 +202,7 @@ public double[] Predict(Matrix X) /// /// The 1D array of predictors. /// Optional. The prng seed. If negative or zero, then the computer clock is used as a seed. - public double[] BootstrapPredict(double[] X, int seed = -1) + public double[]? BootstrapPredict(double[] X, int seed = -1) { return kNNBootstrapPredict(this.X, this.Y, new Matrix(X), seed); } @@ -212,7 +212,7 @@ public double[] BootstrapPredict(double[] X, int seed = -1) /// /// The 2D array of predictors. /// Optional. The prng seed. If negative or zero, then the computer clock is used as a seed. - public double[] BootstrapPredict(double[,] X, int seed = -1) + public double[]? BootstrapPredict(double[,] X, int seed = -1) { return kNNBootstrapPredict(this.X, this.Y, new Matrix(X), seed); } @@ -222,7 +222,7 @@ public double[] BootstrapPredict(double[,] X, int seed = -1) /// /// The matrix of predictors. /// Optional. The prng seed. If negative or zero, then the computer clock is used as a seed. - public double[] BootstrapPredict(Matrix X, int seed = -1) + public double[]? BootstrapPredict(Matrix X, int seed = -1) { return kNNBootstrapPredict(this.X, this.Y, X, seed); } @@ -234,7 +234,7 @@ public double[] BootstrapPredict(Matrix X, int seed = -1) /// Optional. The prng seed. If negative or zero, then the computer clock is used as a seed. /// The number of bootstrap realizations. Default = 1,000. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. - public double[,] PredictionIntervals(double[] X, int seed = -1, int realizations = 1000, double alpha = 0.1) + public double[,]? PredictionIntervals(double[] X, int seed = -1, int realizations = 1000, double alpha = 0.1) { return kNNPredictionIntervals(this.X, this.Y, new Matrix(X), seed, realizations, alpha); } @@ -246,7 +246,7 @@ public double[] BootstrapPredict(Matrix X, int seed = -1) /// Optional. The prng seed. If negative or zero, then the computer clock is used as a seed. /// The number of bootstrap realizations. Default = 1,000. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. - public double[,] PredictionIntervals(double[,] X, int seed = -1, int realizations = 1000, double alpha = 0.1) + public double[,]? PredictionIntervals(double[,] X, int seed = -1, int realizations = 1000, double alpha = 0.1) { return kNNPredictionIntervals(this.X, this.Y, new Matrix(X), seed, realizations, alpha); } @@ -258,7 +258,7 @@ public double[] BootstrapPredict(Matrix X, int seed = -1) /// Optional. The prng seed. If negative or zero, then the computer clock is used as a seed. /// The number of bootstrap realizations. Default = 1,000. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. - public double[,] PredictionIntervals(Matrix X, int seed = -1, int realizations = 1000, double alpha = 0.1) + public double[,]? PredictionIntervals(Matrix X, int seed = -1, int realizations = 1000, double alpha = 0.1) { return kNNPredictionIntervals(this.X, this.Y, X, seed, realizations, alpha); } @@ -269,11 +269,11 @@ public double[] BootstrapPredict(Matrix X, int seed = -1) /// The training matrix of predictors. /// The training response vector. /// The test matrix of predictors - private int[] kNN(Matrix xTrain, Vector yTrain, Matrix xTest) + private int[]? kNN(Matrix xTrain, Vector yTrain, Matrix xTest) { if (NumberOfFeatures != xTrain.NumberOfColumns) return null!; int R = xTest.NumberOfRows; - var result = new int[K]; + var result = new int[R * K]; for (int i = 0; i < R; i++) { var point = xTest.Row(i); @@ -288,10 +288,9 @@ private int[] kNN(Matrix xTrain, Vector yTrain, Matrix xTest) // Sort items and find the k-nearest neighbors Array.Sort(items, (a, b) => a.Distance.CompareTo(b.Distance)); - var knn = new double[K]; for (int j = 0; j < K; j++) { - result[j] = items[j].Index; + result[i * K + j] = items[j].Index; } } @@ -304,7 +303,7 @@ private int[] kNN(Matrix xTrain, Vector yTrain, Matrix xTest) /// The training matrix of predictors. /// The training response vector. /// The test matrix of predictors - private double[] kNNPredict(Matrix xTrain, Vector yTrain, Matrix xTest) + private double[]? kNNPredict(Matrix xTrain, Vector yTrain, Matrix xTest) { if (xTest.NumberOfColumns != xTrain.NumberOfColumns) return null!; int R = xTest.NumberOfRows; @@ -362,7 +361,7 @@ private double[] kNNPredict(Matrix xTrain, Vector yTrain, Matrix xTest) /// The training response vector. /// The test matrix of predictors /// Optional. The prng seed. If negative or zero, then the computer clock is used as a seed. - private double[] kNNBootstrapPredict(Matrix xTrain, Vector yTrain, Matrix xTest, int seed = -1) + private double[]? kNNBootstrapPredict(Matrix xTrain, Vector yTrain, Matrix xTest, int seed = -1) { var rnd = seed > 0 ? new Random(seed) : new Random(); var idxs = rnd.NextIntegers(0, xTrain.NumberOfRows, xTrain.NumberOfRows); @@ -398,7 +397,7 @@ private double[] kNNBootstrapPredict(Matrix xTrain, Vector yTrain, Matrix xTest, var seeds = rnd.NextIntegers(realizations); // Bootstrap the predictions - Parallel.For(0, realizations, idx => { bootResults.SetColumn(idx, kNNBootstrapPredict(xTrain, yTrain, xTest, seeds[idx]));}); + Parallel.For(0, realizations, idx => { bootResults.SetColumn(idx, kNNBootstrapPredict(xTrain, yTrain, xTest, seeds[idx])!);}); // Process results Parallel.For(0, xTest.NumberOfRows, idx => diff --git a/Numerics/Machine Learning/Supervised/NaiveBayes.cs b/Numerics/Machine Learning/Supervised/NaiveBayes.cs index 7a47721b..4ad53997 100644 --- a/Numerics/Machine Learning/Supervised/NaiveBayes.cs +++ b/Numerics/Machine Learning/Supervised/NaiveBayes.cs @@ -143,12 +143,12 @@ public NaiveBayes(Matrix x, Vector y) /// /// The means of each feature given each class. /// - public double[,] Means { get; private set; } = null!; + public double[,] Means { get; private set; } = new double[0, 0]; /// /// The standard deviations of each feature given each class. /// - public double[,] StandardDeviations { get; private set; } = null!; + public double[,] StandardDeviations { get; private set; } = new double[0, 0]; /// /// The prior probability for each class. @@ -213,7 +213,10 @@ public void Train() // Set means Means[i, j] = u1; // Set standard deviations - StandardDeviations[i, j] = Math.Sqrt((u2 - Math.Pow(u1, 2d)) * (n / (n - 1))); + if (n <= 1) + StandardDeviations[i, j] = 1e-6; + else + StandardDeviations[i, j] = Math.Sqrt(Math.Max(0, (u2 - Math.Pow(u1, 2d)) * (n / (n - 1)))); } } @@ -225,7 +228,7 @@ public void Train() /// Returns the prediction of the Naive Bayes classifier /// /// The 1D array of predictors. - public double[] Predict(double[] X) + public double[]? Predict(double[] X) { return Predict(new Matrix(X)); } @@ -234,7 +237,7 @@ public double[] Predict(double[] X) /// Returns the prediction of the Naive Bayes classifier /// /// The 2D array of predictors. - public double[] Predict(double[,] X) + public double[]? Predict(double[,] X) { return Predict(new Matrix(X)); } @@ -243,9 +246,9 @@ public double[] Predict(double[,] X) /// Returns the prediction of the Naive Bayes classifier. /// /// The matrix of predictors. - public double[] Predict(Matrix X) + public double[]? Predict(Matrix X) { - if (!IsTrained || X.NumberOfColumns != this.X.NumberOfColumns) return null!; + if (!IsTrained || X.NumberOfColumns != this.X.NumberOfColumns) return null; var result = new double[X.NumberOfRows]; for (int i = 0; i < X.NumberOfRows; i++) { diff --git a/Numerics/Machine Learning/Supervised/RandomForest.cs b/Numerics/Machine Learning/Supervised/RandomForest.cs index 94da4efe..09cfc171 100644 --- a/Numerics/Machine Learning/Supervised/RandomForest.cs +++ b/Numerics/Machine Learning/Supervised/RandomForest.cs @@ -228,7 +228,7 @@ private DecisionTree BootstrapDecisionTree(int seed = -1) /// /// The 1D array of predictors. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. - public double[,] Predict(double[] X, double alpha = 0.1) + public double[,]? Predict(double[] X, double alpha = 0.1) { return Predict(new Matrix(X), alpha); } @@ -238,7 +238,7 @@ private DecisionTree BootstrapDecisionTree(int seed = -1) /// /// The 2D array of predictors. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. - public double[,] Predict(double[,] X, double alpha = 0.1) + public double[,]? Predict(double[,] X, double alpha = 0.1) { return Predict(new Matrix(X), alpha); } @@ -248,7 +248,7 @@ private DecisionTree BootstrapDecisionTree(int seed = -1) /// /// The matrix of predictors. /// The confidence level; Default = 0.1, which will result in the 90% confidence intervals. - public double[,] Predict(Matrix X, double alpha = 0.1) + public double[,]? Predict(Matrix X, double alpha = 0.1) { if (!IsTrained) return null!; @@ -258,7 +258,7 @@ private DecisionTree BootstrapDecisionTree(int seed = -1) var bootResults = new double[X.NumberOfRows, NumberOfTrees]; // Bootstrap the predictions - Parallel.For(0, NumberOfTrees, idx => { bootResults.SetColumn(idx, DecisionTrees[idx].Predict(X)); }); + Parallel.For(0, NumberOfTrees, idx => { bootResults.SetColumn(idx, DecisionTrees[idx].Predict(X)!); }); // Process results Parallel.For(0, X.NumberOfRows, idx => diff --git a/Numerics/Machine Learning/Support/DecisionNode.cs b/Numerics/Machine Learning/Support/DecisionNode.cs index 4a7d0014..9288d5c6 100644 --- a/Numerics/Machine Learning/Support/DecisionNode.cs +++ b/Numerics/Machine Learning/Support/DecisionNode.cs @@ -54,12 +54,12 @@ public class DecisionNode /// /// Nodes to the left of the threshold. /// - public DecisionNode Left { get; set; } = null!; + public DecisionNode? Left { get; set; } = null; /// /// Nodes to the right of the threshold. /// - public DecisionNode Right { get; set; } = null!; + public DecisionNode? Right { get; set; } = null; /// /// The leaf node value. diff --git a/Numerics/Machine Learning/Support/JenksCluster.cs b/Numerics/Machine Learning/Support/JenksCluster.cs index a2d548b9..2c06c611 100644 --- a/Numerics/Machine Learning/Support/JenksCluster.cs +++ b/Numerics/Machine Learning/Support/JenksCluster.cs @@ -63,21 +63,24 @@ public JenksCluster(double[] data, int startIndex, int endIndex) MinValue = data[startIndex]; MaxValue = data[endIndex]; - // Compute summary statistics - double X = 0; // sum - double X2 = 0; // sum of X^2 - for (int i = startIndex; i <= endIndex; i++) + // Compute summary statistics using Welford's algorithm for numerical stability + double sum = 0; + double mean = 0; + double m2 = 0; + for (int i = startIndex; i <= endIndex; i++) { - X += data[i]; - X2 += Math.Pow(data[i], 2d); + sum += data[i]; + int k = i - startIndex + 1; + double delta = data[i] - mean; + mean += delta / k; + double delta2 = data[i] - mean; + m2 += delta * delta2; } - double U1 = X / Count; - double U2 = X2 / Count; - Sum = X; - Average = Count == 1 ? X : U1; - Variance = Count == 1 ? 0 : (U2 - Math.Pow(U1, 2d)) * (Count / (double)(Count - 1)); - SumOfSquaredDeviations = Variance * (double)(Count - 1); + Sum = sum; + Average = mean; + SumOfSquaredDeviations = m2; + Variance = Count <= 1 ? 0 : m2 / (Count - 1); } /// diff --git a/Numerics/Machine Learning/Unsupervised/GaussianMixtureModel.cs b/Numerics/Machine Learning/Unsupervised/GaussianMixtureModel.cs index 25ccdeb1..33a405df 100644 --- a/Numerics/Machine Learning/Unsupervised/GaussianMixtureModel.cs +++ b/Numerics/Machine Learning/Unsupervised/GaussianMixtureModel.cs @@ -157,7 +157,7 @@ public GaussianMixtureModel(Matrix X, int k) /// /// The likelihood of each data point (row) and for each cluster (column). /// - public double[,] LikelihoodMatrix { get; private set; } = null!; + public double[,] LikelihoodMatrix { get; private set; } = new double[0, 0]; /// /// The total log-likelihood of the fit. @@ -296,14 +296,14 @@ private void MStep() for (int k = 0; k < K; k++) { double wgt = 0d; - for (int i = 0; i < X.NumberOfRows; i++) + for (int i = 0; i < X.NumberOfRows; i++) wgt += LikelihoodMatrix[i, k]; Weights[k] = wgt / X.NumberOfRows; for (int d = 0; d < Dimension; d++) { // Compute centroids double sum = 0; - for (int i = 0; i < X.NumberOfRows; i++) + for (int i = 0; i < X.NumberOfRows; i++) sum += LikelihoodMatrix[i, k] * X[i, d]; Means[k, d] = sum / wgt; // Compute covariance @@ -317,6 +317,14 @@ private void MStep() Sigmas[k][d, j] = sum / wgt; } } + + // Add small regularization to the diagonal to ensure the covariance + // matrix remains positive-definite. When a component collapses to very + // few points, the covariance can become singular, causing Cholesky + // decomposition in the E-step to fail. + for (int d = 0; d < Dimension; d++) + MatrixRegularization.MakeSymmetricPositiveDefinite(Sigmas[k]); + } } diff --git a/Numerics/Machine Learning/Unsupervised/KMeans.cs b/Numerics/Machine Learning/Unsupervised/KMeans.cs index 00989712..f4abcb20 100644 --- a/Numerics/Machine Learning/Unsupervised/KMeans.cs +++ b/Numerics/Machine Learning/Unsupervised/KMeans.cs @@ -241,7 +241,7 @@ public void Train(int seed = -1, bool kMeansPlusPlus= true) double min = Tools.Distance(x, centroids.GetRow(0)); for (int j = 1; j < c; j++) { - double d = Tools.Distance(x, centroids.GetRow(0)); + double d = Tools.Distance(x, centroids.GetRow(j)); if (d < min) min = d; diff --git a/Numerics/Mathematics/Fourier Methods/Fourier.cs b/Numerics/Mathematics/Fourier Methods/Fourier.cs index 922a56ce..5d17983f 100644 --- a/Numerics/Mathematics/Fourier Methods/Fourier.cs +++ b/Numerics/Mathematics/Fourier Methods/Fourier.cs @@ -277,7 +277,7 @@ public static double[] Correlation(double[] data1, double[] data2) /// /// A 2-column array with the first column contains the lags, and the second the autocorrelation. /// - public static double[,] Autocorrelation(IList series, int lagMax = -1) + public static double[,]? Autocorrelation(IList series, int lagMax = -1) { int n = series.Count; if (lagMax < 0) diff --git a/Numerics/Mathematics/Integration/AdaptiveGuassKronrod.cs b/Numerics/Mathematics/Integration/AdaptiveGuassKronrod.cs index f942fd0b..9f938447 100644 --- a/Numerics/Mathematics/Integration/AdaptiveGuassKronrod.cs +++ b/Numerics/Mathematics/Integration/AdaptiveGuassKronrod.cs @@ -66,7 +66,7 @@ public class AdaptiveGaussKronrod : Integrator /// The maximum value under which the integral must be computed. public AdaptiveGaussKronrod(Func function, double min, double max) { - if (max <= min) throw new ArgumentNullException(nameof(max), "The maximum value cannot be less than or equal to the minimum value."); + if (max <= min) throw new ArgumentOutOfRangeException(nameof(max), "The maximum value cannot be less than or equal to the minimum value."); Function = function ?? throw new ArgumentNullException(nameof(function), "The function cannot be null."); a = min; b = max; diff --git a/Numerics/Mathematics/Integration/Integration.cs b/Numerics/Mathematics/Integration/Integration.cs index 4ad19df6..1f12f7ab 100644 --- a/Numerics/Mathematics/Integration/Integration.cs +++ b/Numerics/Mathematics/Integration/Integration.cs @@ -72,6 +72,46 @@ public static double GaussLegendre(Func f, double a, double b) return s *= xr; } + /// + /// Returns the integral of a function between a and b by twenty-point Gauss-Legendre integration. + /// + /// The function to integrate. + /// Start point for integration. + /// End point for integration. + /// The value of a definite integral. + /// + /// Twenty-point Gauss-Legendre quadrature is exact for polynomials of degree 39 or less. + /// Uses 10 symmetric node pairs (20 function evaluations total). Provides higher accuracy + /// than the 10-point method for non-polynomial smooth integrands + /// such as log-transformed functions. + /// + /// Nodes are roots of the Legendre polynomial P₂₀(x); weights are the corresponding + /// Christoffel numbers. Reference: Abramowitz and Stegun (1964), Table 25.4. + /// + /// + public static double GaussLegendre20(Func f, double a, double b) + { + var x = new double[] { + 0.0765265211334973338, 0.2277858511416450781, 0.3737060887154195607, + 0.5108670019508270981, 0.6360536807265150254, 0.7463319064601507926, + 0.8391169718222188234, 0.9122344282513259059, 0.9639719272779137912, + 0.9931285991850949247 }; + var w = new double[] { + 0.1527533871307258507, 0.1491729864726037467, 0.1420961093183820514, + 0.1316886384491766269, 0.1181945319615184174, 0.1019301198172404351, + 0.0832767415767047487, 0.0626720483341090636, 0.0406014298003869413, + 0.0176140071391521183 }; + double xm = 0.5 * (b + a); + double xr = 0.5 * (b - a); + double s = 0; + for (int j = 0; j < 10; j++) + { + double dx = xr * x[j]; + s += w[j] * (f(xm + dx) + f(xm - dx)); + } + return s *= xr; + } + /// /// Numerical integration using the Trapezoidal Rule. /// diff --git a/Numerics/Mathematics/Integration/Miser.cs b/Numerics/Mathematics/Integration/Miser.cs index 215feddf..d35aa105 100644 --- a/Numerics/Mathematics/Integration/Miser.cs +++ b/Numerics/Mathematics/Integration/Miser.cs @@ -340,12 +340,10 @@ private void miser(Func function, double[] regn, int npts, dou private void ranpt(double[] pt, double[] regn) { int j, n = pt.Length; - double[] rnd = null!; - if (UseSobolSequence) - rnd = _sobol.NextDouble(); + double[]? rnd = UseSobolSequence ? _sobol.NextDouble() : null; for (j = 0; j < n; j++) - pt[j] = regn[j] + (regn[n + j] - regn[j]) * (UseSobolSequence ? rnd[j] : Random.NextDouble()); + pt[j] = regn[j] + (regn[n + j] - regn[j]) * (rnd != null ? rnd[j] : Random.NextDouble()); } } } diff --git a/Numerics/Mathematics/Integration/Vegas.cs b/Numerics/Mathematics/Integration/Vegas.cs index 39205954..5a522952 100644 --- a/Numerics/Mathematics/Integration/Vegas.cs +++ b/Numerics/Mathematics/Integration/Vegas.cs @@ -116,7 +116,7 @@ public Vegas(Func function, int dimensions, IList /// Gets the number of rows. diff --git a/Numerics/Mathematics/Linear Algebra/Support/Vector.cs b/Numerics/Mathematics/Linear Algebra/Support/Vector.cs index b72e1416..86eaaec0 100644 --- a/Numerics/Mathematics/Linear Algebra/Support/Vector.cs +++ b/Numerics/Mathematics/Linear Algebra/Support/Vector.cs @@ -108,9 +108,9 @@ public double this[int index] set { _vector[index] = value; } } - /// - /// The vector header text. - /// + /// + /// The vector header text. + /// public string Header { get; set; } = null!; diff --git a/Numerics/Mathematics/Optimization/Constrained/AugmentedLagrange.cs b/Numerics/Mathematics/Optimization/Constrained/AugmentedLagrange.cs index bec630bb..b724963f 100644 --- a/Numerics/Mathematics/Optimization/Constrained/AugmentedLagrange.cs +++ b/Numerics/Mathematics/Optimization/Constrained/AugmentedLagrange.cs @@ -122,34 +122,37 @@ public AugmentedLagrange(Func objectiveFunction, Optimizer opt public ReadOnlyCollection Nu => new ReadOnlyCollection(_nu); /// - /// The Augmented Lagrangian objective function. + /// The Augmented Lagrangian objective function. /// private double augmentedLagrangianFunction(double[] x) { double phi = _primaryObjectiveFunction(x); double rho2 = 0.5 * rho; - // For each equality constraint + int lambdaIdx = 0, muIdx = 0, nuIdx = 0; for (int i = 0; i < _constraints.Length; i++) { double actual = _constraints[i].Function(x); double c = 0; - + switch (_constraints[i].Type) { case ConstraintType.EqualTo: c = actual - _constraints[i].Value; - phi += rho2 * Math.Pow(c + _lambda[i] / rho, 2d); + phi += rho2 * Math.Pow(c + _lambda[lambdaIdx] / rho, 2d); + lambdaIdx++; break; case ConstraintType.LesserThanOrEqualTo: c = actual - _constraints[i].Value; - if (c > 0) phi += rho2 * Math.Pow(c + _mu[i] / rho, 2d); + if (c > 0) phi += rho2 * Math.Pow(c + _mu[muIdx] / rho, 2d); + muIdx++; break; case ConstraintType.GreaterThanOrEqualTo: c = _constraints[i].Value - actual; - if (c > 0) phi += rho2 * Math.Pow(c + _nu[i] / rho, 2d); + if (c > 0) phi += rho2 * Math.Pow(c + _nu[nuIdx] / rho, 2d); + nuIdx++; break; } } @@ -263,6 +266,7 @@ protected override void Optimize() bool feasible = true; // Update lambdas + int lambdaIdx = 0, muIdx = 0, nuIdx = 0; for (int i = 0; i < _constraints.Length; i++) { double actual = _constraints[i].Function(currentValues); @@ -273,29 +277,32 @@ protected override void Optimize() { case ConstraintType.EqualTo: c = actual - _constraints[i].Value; - newLambda = _lambda[i] + rho * c; + newLambda = _lambda[lambdaIdx] + rho * c; penalty += Math.Abs(c); feasible = feasible && Math.Abs(c) <= _constraints[i].Tolerance; ICM = Math.Max(ICM, Math.Abs(c)); - _lambda[i] = Math.Min(Math.Max(-1e20, newLambda), 1e20); + _lambda[lambdaIdx] = Math.Min(Math.Max(-1e20, newLambda), 1e20); + lambdaIdx++; break; case ConstraintType.LesserThanOrEqualTo: c = actual - _constraints[i].Value; - newLambda = _mu[i] + rho * c; + newLambda = _mu[muIdx] + rho * c; penalty += c > 0 ? c : 0; feasible = feasible && c <= _constraints[i].Tolerance; - ICM = Math.Max(ICM, Math.Abs(Math.Max(c, -_mu[i] / rho))); - _mu[i] = Math.Min(Math.Max(0.0, newLambda), 1e20); + ICM = Math.Max(ICM, Math.Abs(Math.Max(c, -_mu[muIdx] / rho))); + _mu[muIdx] = Math.Min(Math.Max(0.0, newLambda), 1e20); + muIdx++; break; case ConstraintType.GreaterThanOrEqualTo: c = _constraints[i].Value - actual; - newLambda = _nu[i] + rho * c; + newLambda = _nu[nuIdx] + rho * c; penalty += c > 0 ? c : 0; feasible = feasible && c <= _constraints[i].Tolerance; - ICM = Math.Max(ICM, Math.Abs(Math.Max(c, -_nu[i] / rho))); - _nu[i] = Math.Min(Math.Max(0.0, newLambda), 1e20); + ICM = Math.Max(ICM, Math.Abs(Math.Max(c, -_nu[nuIdx] / rho))); + _nu[nuIdx] = Math.Min(Math.Max(0.0, newLambda), 1e20); + nuIdx++; break; } } diff --git a/Numerics/Mathematics/Optimization/Dynamic/BinaryHeap.cs b/Numerics/Mathematics/Optimization/Dynamic/BinaryHeap.cs index cec0c80a..76fde4b1 100644 --- a/Numerics/Mathematics/Optimization/Dynamic/BinaryHeap.cs +++ b/Numerics/Mathematics/Optimization/Dynamic/BinaryHeap.cs @@ -45,12 +45,30 @@ namespace Numerics.Mathematics.Optimization /// Generic variable to store with each node. Typically used to store important data associated with the network that isn't required for the binary heap. public class BinaryHeap { + /// + /// Represents a node in the binary heap with a weight, index, and value. + /// public struct Node { + /// + /// The weight (priority) of the node. + /// public float Weight; + /// + /// The index identifier of the node. + /// public int Index; + /// + /// The value stored in the node. + /// public T Value; + /// + /// Creates a new node with the specified weight, index, and value. + /// + /// The weight (priority) of the node. + /// The index identifier of the node. + /// The value to store in the node. public Node(float nodeWeight, int nodeIndex, T nodeValue) { Weight = nodeWeight; @@ -65,8 +83,15 @@ public Node(float nodeWeight, int nodeIndex, T nodeValue) private int _n = 0; // Number of nodes. //private int _p = 0; // Parent Index + /// + /// The number of nodes in the heap. + /// public int Count => _n; + /// + /// Creates a new binary heap with the specified maximum size. + /// + /// The maximum number of nodes the heap can hold. public BinaryHeap(int heapSize) { _heap = new Node[heapSize]; diff --git a/Numerics/Mathematics/Optimization/Dynamic/Dijkstra.cs b/Numerics/Mathematics/Optimization/Dynamic/Dijkstra.cs index 05197518..01f23195 100644 --- a/Numerics/Mathematics/Optimization/Dynamic/Dijkstra.cs +++ b/Numerics/Mathematics/Optimization/Dynamic/Dijkstra.cs @@ -94,7 +94,7 @@ public static bool PathExists(float[,] resultTable, int nodeIndex) /// Optional number of nodes in the network. If not provided it will be calculated internally. /// Optional list of incoming edges from each node in the network. If not provided or mismatched with edges it will be calculated internally. /// Lookup table of shortest paths from any given node. - public static float[,] Solve(IList edges, int[] destinationIndices, int nodeCount = -1, List[] edgesFromNodes = null!) + public static float[,] Solve(IList edges, int[] destinationIndices, int nodeCount = -1, List[]? edgesFromNodes = null) { // Set optional parameters if required. int nNodes = (nodeCount == -1) ? (edges.Max(o => Math.Max(o.FromIndex,o.ToIndex)) + 1) : nodeCount; @@ -147,7 +147,7 @@ public static bool PathExists(float[,] resultTable, int nodeIndex) /// Optional number of nodes in the network. If not provided it will be calculated internally. /// Optional list of incoming edges from each node in the network. If not provided or mismatched with edges it will be calculated internally. /// Lookup table of shortest paths from any given node. - public static float[,] Solve(IList edges, int destinationIndex, int nodeCount = -1, List[] edgesToNodes = null!) + public static float[,] Solve(IList edges, int destinationIndex, int nodeCount = -1, List[]? edgesToNodes = null) { // Set optional parameters if required. int nNodes = (nodeCount == -1) ? (edges.Max(o => Math.Max(o.FromIndex, o.ToIndex)) + 1) : nodeCount; diff --git a/Numerics/Mathematics/Optimization/Dynamic/Network.cs b/Numerics/Mathematics/Optimization/Dynamic/Network.cs index 6c4968b6..b433fa8c 100644 --- a/Numerics/Mathematics/Optimization/Dynamic/Network.cs +++ b/Numerics/Mathematics/Optimization/Dynamic/Network.cs @@ -48,12 +48,26 @@ public class Network //private readonly RoadSegment[] _segments; private readonly Edge[] _edges; - //public RoadSegment[] Segments { get => _segments; } + /// + /// The destination node indices for shortest path computation. + /// public int[] DestinationIndices { get => _destinationIndices; } + /// + /// The incoming edges for each node, indexed by node index. + /// public List[] IncomingEdges { get => _incomingEdges; } + + /// + /// The outgoing edges for each node, indexed by node index. + /// public List[] OutgoingEdges { get => _outgoingEdges; } + /// + /// Creates a new network from the specified edges and destination indices. + /// + /// The edges that define the network. + /// The destination node indices. public Network(Edge[] edges, int[] destinationIndices) { //_segments = roadSegments; @@ -88,16 +102,31 @@ public Network(Edge[] edges, int[] destinationIndices) } + /// + /// Solves the shortest path from all nodes to the specified destination. + /// + /// The destination node index. + /// A result table with predecessor, edge index, and cumulative weight for each node. public float[,] Solve(int destinationIndex) { return Dijkstra.Solve(_edges, destinationIndex, _nodeCount, _incomingEdges); } + /// + /// Solves the shortest path from all nodes to the specified destinations. + /// + /// The destination node indices. + /// A result table with predecessor, edge index, and cumulative weight for each node. public float[,] Solve(int[] destinationIndices) { return Dijkstra.Solve(_edges, destinationIndices, _nodeCount, _incomingEdges); } + /// + /// Solves the shortest path using custom edge weights. + /// + /// Custom weights for each edge. + /// A result table with predecessor, edge index, and cumulative weight for each node. public float[,] Solve(float[] edgeWeights) { Edge[] edges = new Edge[_edges.Length]; @@ -109,7 +138,13 @@ public Network(Edge[] edges, int[] destinationIndices) return Dijkstra.Solve(edges, _destinationIndices, _nodeCount, _incomingEdges); } - public List GetPath(int[] edgesToRemove, int startNodeIndex) + /// + /// Finds an alternative path avoiding the specified edges. + /// + /// Edge indices to exclude from the path. + /// The starting node index. + /// A list of edge indices forming the alternative path, or null if no path exists. + public List? GetPath(int[] edgesToRemove, int startNodeIndex) { int[] nodeState = new int[_nodeCount]; float[] nodeWeightToDestination = new float[_nodeCount]; @@ -348,7 +383,14 @@ public List GetPath(int[] edgesToRemove, int startNodeIndex) else return null!; } - public List GetPath(int[] edgesToRemove, int startNodeIndex, float[,] existingResultsTable) + /// + /// Finds an alternative path avoiding the specified edges, using a pre-computed results table. + /// + /// Edge indices to exclude from the path. + /// The starting node index. + /// A pre-computed shortest path results table. + /// A list of edge indices forming the alternative path, or an empty list if no path exists. + public List? GetPath(int[] edgesToRemove, int startNodeIndex, float[,] existingResultsTable) { int[] nodeState = new int[_nodeCount]; float[] nodeWeightToDestination = new float[_nodeCount]; diff --git a/Numerics/Mathematics/Optimization/Global/MLSL.cs b/Numerics/Mathematics/Optimization/Global/MLSL.cs index 01c066bf..90e21695 100644 --- a/Numerics/Mathematics/Optimization/Global/MLSL.cs +++ b/Numerics/Mathematics/Optimization/Global/MLSL.cs @@ -120,17 +120,17 @@ public MLSL(Func objectiveFunction, int numberOfParameters, IL /// /// An array of initial values to evaluate. /// - public double[] InitialValues { get; private set; } + public double[] InitialValues { get; private set; } = null!; /// - /// An array of lower bounds (inclusive) of the interval containing the optimal point. + /// An array of lower bounds (inclusive) of the interval containing the optimal point. /// - public double[] LowerBounds { get; private set; } + public double[] LowerBounds { get; private set; } = null!; /// /// An array of upper bounds (inclusive) of the interval containing the optimal point. /// - public double[] UpperBounds { get; private set; } + public double[] UpperBounds { get; private set; } = null!; /// /// The pseudo random number generator (PRNG) seed. @@ -207,7 +207,7 @@ protected override void Optimize() double oldFit = double.MaxValue; int noImprovement = 0; bool cancel = false; - Optimizer solver = null!; + Optimizer? solver = null; var prng = new MersenneTwister(PRNGSeed); // Set lower and upper bounds and @@ -378,7 +378,7 @@ protected override void Optimize() private Optimizer GetLocalOptimizer(IList initialValues, double relativeTolerance, double absoluteTolerance, ref bool cancel) { bool localCancel = false; - Optimizer solver = null!; + Optimizer? solver = null; // Make sure the parameters are within the bounds. for (int i = 0; i < NumberOfParameters; i++) diff --git a/Numerics/Mathematics/Optimization/Global/MultiStart.cs b/Numerics/Mathematics/Optimization/Global/MultiStart.cs index 05ac9006..28335f61 100644 --- a/Numerics/Mathematics/Optimization/Global/MultiStart.cs +++ b/Numerics/Mathematics/Optimization/Global/MultiStart.cs @@ -163,7 +163,7 @@ protected override void Optimize() { int i, j, D = NumberOfParameters; bool cancel = false; - Optimizer solver = null!; + Optimizer? solver = null; // Set lower and upper bounds and // create uniform distributions for each parameter @@ -219,7 +219,7 @@ protected override void Optimize() private Optimizer GetLocalOptimizer(IList initialValues, double relativeTolerance, double absoluteTolerance, ref bool cancel) { bool localCancel = false; - Optimizer solver = null!; + Optimizer? solver = null; // Make sure the parameters are within the bounds. for (int i = 0; i < NumberOfParameters; i++) diff --git a/Numerics/Mathematics/Optimization/Global/ParticleSwarm.cs b/Numerics/Mathematics/Optimization/Global/ParticleSwarm.cs index 566664f4..3384822c 100644 --- a/Numerics/Mathematics/Optimization/Global/ParticleSwarm.cs +++ b/Numerics/Mathematics/Optimization/Global/ParticleSwarm.cs @@ -95,12 +95,12 @@ public ParticleSwarm(Func objectiveFunction, int numberOfParam /// /// An array of lower bounds (inclusive) of the interval containing the optimal point. /// - public double[] LowerBounds { get; private set; } + public double[] LowerBounds { get; private set; } = null!; /// /// An array of upper bounds (inclusive) of the interval containing the optimal point. /// - public double[] UpperBounds { get; private set; } + public double[] UpperBounds { get; private set; } = null!; /// /// The total population size. Default = 30. diff --git a/Numerics/Mathematics/Optimization/Global/SimulatedAnnealing.cs b/Numerics/Mathematics/Optimization/Global/SimulatedAnnealing.cs index 250bf011..f889b915 100644 --- a/Numerics/Mathematics/Optimization/Global/SimulatedAnnealing.cs +++ b/Numerics/Mathematics/Optimization/Global/SimulatedAnnealing.cs @@ -120,6 +120,9 @@ public SimulatedAnnealing(Func objectiveFunction, int numberOf /// public double MinTemperature { get; set; } = 0.1; + /// + /// The cooling rate for the annealing schedule. Default = 0.95. + /// public double CoolingRate { get; set; } = 0.95; /// diff --git a/Numerics/Mathematics/Optimization/Local/ADAM.cs b/Numerics/Mathematics/Optimization/Local/ADAM.cs index 0d8466ec..822cab3f 100644 --- a/Numerics/Mathematics/Optimization/Local/ADAM.cs +++ b/Numerics/Mathematics/Optimization/Local/ADAM.cs @@ -76,7 +76,7 @@ public class ADAM : Optimizer /// Optional. Function to evaluate the gradient. Default uses finite difference. public ADAM(Func objectiveFunction, int numberOfParameters, IList initialValues, IList lowerBounds, IList upperBounds, double alpha = 0.001, - Func gradient = null!) : base(objectiveFunction, numberOfParameters) + Func? gradient = null) : base(objectiveFunction, numberOfParameters) { // Check if the length of the initial, lower and upper bounds equal the number of parameters if (initialValues.Count != numberOfParameters || lowerBounds.Count != numberOfParameters || upperBounds.Count != numberOfParameters) @@ -135,7 +135,7 @@ public ADAM(Func objectiveFunction, int numberOfParameters, /// /// The function for evaluating the gradient of the objective function. /// - public Func Gradient; + public Func? Gradient; /// protected override void Optimize() diff --git a/Numerics/Mathematics/Optimization/Local/BFGS.cs b/Numerics/Mathematics/Optimization/Local/BFGS.cs index 90f949a0..a7b02d54 100644 --- a/Numerics/Mathematics/Optimization/Local/BFGS.cs +++ b/Numerics/Mathematics/Optimization/Local/BFGS.cs @@ -78,7 +78,7 @@ public class BFGS : Optimizer /// Optional. Function to evaluate the gradient. Default uses finite difference. public BFGS(Func objectiveFunction, int numberOfParameters, IList initialValues, IList lowerBounds, IList upperBounds, - Func gradient = null!) : base(objectiveFunction, numberOfParameters) + Func? gradient = null) : base(objectiveFunction, numberOfParameters) { // Check if the length of the initial, lower and upper bounds equal the number of parameters if (initialValues.Count != numberOfParameters || lowerBounds.Count != numberOfParameters || upperBounds.Count != numberOfParameters) @@ -121,7 +121,7 @@ public BFGS(Func objectiveFunction, int numberOfParameters, /// /// The function for evaluating the gradient of the objective function. /// - public Func Gradient; + public Func? Gradient; /// protected override void Optimize() diff --git a/Numerics/Mathematics/Optimization/Local/GradientDescent.cs b/Numerics/Mathematics/Optimization/Local/GradientDescent.cs index d19320af..494dcfaf 100644 --- a/Numerics/Mathematics/Optimization/Local/GradientDescent.cs +++ b/Numerics/Mathematics/Optimization/Local/GradientDescent.cs @@ -77,7 +77,7 @@ public class GradientDescent : Optimizer /// Optional. Function to evaluate the gradient. Default uses finite difference. public GradientDescent(Func objectiveFunction, int numberOfParameters, IList initialValues, IList lowerBounds, IList upperBounds, double alpha = 0.001, - Func gradient = null!) : base(objectiveFunction, numberOfParameters) + Func? gradient = null) : base(objectiveFunction, numberOfParameters) { // Check if the length of the initial, lower and upper bounds equal the number of parameters if (initialValues.Count != numberOfParameters || lowerBounds.Count != numberOfParameters || upperBounds.Count != numberOfParameters) @@ -126,7 +126,7 @@ public GradientDescent(Func objectiveFunction, int numberOfPar /// /// The function for evaluating the gradient of the objective function. /// - public Func Gradient; + public Func? Gradient; /// protected override void Optimize() diff --git a/Numerics/Mathematics/Optimization/Support/Optimizer.cs b/Numerics/Mathematics/Optimization/Support/Optimizer.cs index f5605d56..d2fb95c3 100644 --- a/Numerics/Mathematics/Optimization/Support/Optimizer.cs +++ b/Numerics/Mathematics/Optimization/Support/Optimizer.cs @@ -137,22 +137,22 @@ public Func ObjectiveFunction /// /// The optimal point, or parameter set. /// - public ParameterSet BestParameterSet { get; protected set; } + public ParameterSet BestParameterSet { get; protected set; } = new ParameterSet(); /// /// A trace of the parameter set and fitness evaluated until convergence. /// - public List ParameterSetTrace { get; protected set; } = null!; + public List ParameterSetTrace { get; protected set; } = new List(); /// - /// Determines the optimization method status. + /// Determines the optimization method status. /// public OptimizationStatus Status { get; protected set; } = OptimizationStatus.None; /// /// The numerically differentiated Hessian matrix. This is only computed when the optimization is successful. /// - public Matrix Hessian { get; protected set; } = null!; + public Matrix? Hessian { get; protected set; } #endregion @@ -166,7 +166,7 @@ public virtual void ClearResults() BestParameterSet = new ParameterSet(); ParameterSetTrace = new List(); Status = OptimizationStatus.None; - Hessian = null!; + Hessian = null; } /// @@ -287,7 +287,7 @@ protected virtual double RepairParameter(double value, double lowerBound, double /// /// Optimization status. /// Inner exception. - protected virtual void UpdateStatus(OptimizationStatus status, Exception exception = null!) + protected virtual void UpdateStatus(OptimizationStatus status, Exception? exception = null) { Status = status; if (status == OptimizationStatus.MaximumIterationsReached) diff --git a/Numerics/Mathematics/Optimization/Support/ParameterSet.cs b/Numerics/Mathematics/Optimization/Support/ParameterSet.cs index 86a61141..dd122a66 100644 --- a/Numerics/Mathematics/Optimization/Support/ParameterSet.cs +++ b/Numerics/Mathematics/Optimization/Support/ParameterSet.cs @@ -94,11 +94,10 @@ public ParameterSet(XElement xElement) Values[i] = outVal; } } - - var fitAttr = xElement.Attribute(nameof(Fitness)); - if (fitAttr != null) + var fitnessAttr = xElement.Attribute(nameof(Fitness)); + if (fitnessAttr != null) { - double.TryParse(fitAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var fitness); + double.TryParse(fitnessAttr.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var fitness); Fitness = fitness; } var weightAttr = xElement.Attribute(nameof(Weight)); diff --git a/Numerics/Mathematics/Root Finding/Brent.cs b/Numerics/Mathematics/Root Finding/Brent.cs index e065241e..1ed59a13 100644 --- a/Numerics/Mathematics/Root Finding/Brent.cs +++ b/Numerics/Mathematics/Root Finding/Brent.cs @@ -191,7 +191,7 @@ public static double Solve(Func f, double lowerBound, double upp } else { - return root; + return solutionFound ? root : b; } } @@ -210,8 +210,6 @@ public static bool Bracket(Func f, ref double lowerBound, ref do if (lowerBound == upperBound) throw new Exception("Bad initial range in bracket."); f1 = f(lowerBound); f2 = f(upperBound); - - if (lowerBound == upperBound) throw new Exception("Bad initial range in bracket."); for (int j = 0; j < maxIterations; j++) { diff --git a/Numerics/Mathematics/Special Functions/Bessel.cs b/Numerics/Mathematics/Special Functions/Bessel.cs new file mode 100644 index 00000000..be75e803 --- /dev/null +++ b/Numerics/Mathematics/Special Functions/Bessel.cs @@ -0,0 +1,691 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; + +namespace Numerics.Mathematics.SpecialFunctions +{ + + /// + /// Contains methods for evaluating Bessel functions of the first kind (J), second kind (Y), + /// modified Bessel functions of the first kind (I), and modified Bessel functions of the second kind (K). + /// + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// Description: + /// + /// + /// Bessel functions are solutions to Bessel's differential equation: + /// + /// x² y'' + x y' + (x² - n²) y = 0 + /// + /// They arise naturally in problems with cylindrical or spherical symmetry, including heat conduction, + /// wave propagation, electrostatics, and fluid dynamics. In statistics, the modified Bessel functions + /// I₀ and I₁ appear in the von Mises distribution for circular data. + /// + /// + /// This class provides the following functions: + /// + /// J₀(x), J₁(x), Jₙ(x): Bessel functions of the first kind. Solutions regular at the origin. + /// Y₀(x), Y₁(x), Yₙ(x): Bessel functions of the second kind (Neumann functions). Singular at the origin. + /// I₀(x), I₁(x), Iₙ(x): Modified Bessel functions of the first kind. Exponentially growing solutions of the modified equation. + /// K₀(x), K₁(x), Kₙ(x): Modified Bessel functions of the second kind. Exponentially decaying solutions of the modified equation. + /// + /// + /// + /// For orders 0 and 1, rational polynomial and polynomial approximations are used directly. For arbitrary + /// integer orders, stable recurrence relations are employed: forward recurrence for Y and K (which are + /// stable in the forward direction), and Miller's downward recurrence for J and I (which are stable in the + /// backward direction), with normalization against the known order-0 values. + /// + /// + /// References: + /// + /// + /// + /// + /// Abramowitz, M. and Stegun, I.A. (1972). "Handbook of Mathematical Functions." + /// National Bureau of Standards, Applied Mathematics Series 55. Sections 9.1-9.8. + /// + /// + /// Press, W.H., Teukolsky, S.A., Vetterling, W.T. and Flannery, B.P. (2007). + /// "Numerical Recipes: The Art of Scientific Computing," 3rd ed. Cambridge University Press. + /// Sections 6.5-6.7. + /// + /// + /// Olver, F.W.J. et al. (2010). "NIST Handbook of Mathematical Functions." + /// Cambridge University Press. Chapter 10. + /// + /// + /// + /// + public static class Bessel + { + + /// + /// Accuracy parameter for Miller's downward recurrence. Larger values give more accurate results + /// at the cost of more iterations. A value of 40 gives approximately double precision accuracy. + /// + private const int IACC = 40; + + /// + /// Rescaling threshold for preventing overflow during Miller's downward recurrence. + /// + private const double BIGNO = 1.0e10; + + /// + /// Inverse of BIGNO for rescaling during Miller's downward recurrence. + /// + private const double BIGNI = 1.0e-10; + + #region Modified Bessel Functions of the First Kind (I) + + /// + /// Computes the modified Bessel function of the first kind, order 0: I₀(x). + /// + /// The argument at which to evaluate I₀. Can be any real number. + /// The value of I₀(x). Always positive. I₀(x) = I₀(-x). + /// + /// + /// Uses polynomial approximations from Abramowitz and Stegun (1972), sections 9.8.1 and 9.8.2. + /// For |x| <= 3.75, uses a polynomial in (x/3.75)² with maximum error |e| < 1.6x10⁻⁷. + /// For |x| > 3.75, uses an asymptotic expansion with maximum error |e| < 1.9x10⁻⁷. + /// + /// + /// I₀(x) is the zeroth-order modified Bessel function of the first kind, satisfying: + /// + /// x² y'' + x y' - x² y = 0 + /// + /// It is related to J₀ by I₀(x) = J₀(ix), and appears as the normalization constant + /// in the von Mises distribution: f(x|mu,kappa) = exp(kappa*cos(x-mu)) / (2*pi*I₀(kappa)). + /// + /// + public static double I0(double x) + { + double ax = Math.Abs(x); + if (ax < 3.75) + { + double t = x / 3.75; + t *= t; + return 1.0 + t * (3.5156229 + t * (3.0899424 + t * (1.2067492 + + t * (0.2659732 + t * (0.0360768 + t * 0.0045813))))); + } + else + { + if (ax > 709) return double.PositiveInfinity; + double t = 3.75 / ax; + return (Math.Exp(ax) / Math.Sqrt(ax)) * (0.39894228 + t * (0.01328592 + + t * (0.00225319 + t * (-0.00157565 + t * (0.00916281 + t * (-0.02057706 + + t * (0.02635537 + t * (-0.01647633 + t * 0.00392377)))))))); + } + } + + /// + /// Computes the modified Bessel function of the first kind, order 1: I₁(x). + /// + /// The argument at which to evaluate I₁. Can be any real number. + /// The value of I₁(x). I₁(-x) = -I₁(x) (odd function). + /// + /// + /// Uses polynomial approximations from Abramowitz and Stegun (1972), sections 9.8.3 and 9.8.4. + /// For |x| <= 3.75, uses a polynomial in (x/3.75)² with maximum error |e| < 8x10⁻⁹. + /// For |x| > 3.75, uses an asymptotic expansion with maximum error |e| < 2.2x10⁻⁷. + /// + /// + /// I₁(x) appears in circular statistics: the mean resultant length of the von Mises distribution + /// is A(kappa) = I₁(kappa)/I₀(kappa), and the circular variance is V = 1 - A(kappa). + /// + /// + public static double I1(double x) + { + double ax = Math.Abs(x); + double result; + if (ax < 3.75) + { + double t = x / 3.75; + t *= t; + result = ax * (0.5 + t * (0.87890594 + t * (0.51498869 + t * (0.15084934 + + t * (0.02658733 + t * (0.00301532 + t * 0.00032411)))))); + } + else + { + double t = 3.75 / ax; + result = (Math.Exp(ax) / Math.Sqrt(ax)) * (0.39894228 + t * (-0.03988024 + + t * (-0.00362018 + t * (0.00163801 + t * (-0.01031555 + t * (0.02282967 + + t * (-0.02895312 + t * (0.01787654 + t * (-0.00420059))))))))); + } + return x < 0 ? -result : result; + } + + /// + /// Computes the modified Bessel function of the first kind for integer order n: Iₙ(x). + /// + /// The order of the function. Must be non-negative (n >= 0). + /// The argument at which to evaluate Iₙ. Can be any real number. + /// + /// The value of Iₙ(x). For odd n, Iₙ(-x) = -Iₙ(x). For even n, Iₙ(-x) = Iₙ(x). + /// + /// + /// + /// For n = 0 and n = 1, delegates to and respectively. + /// For n >= 2, uses Miller's downward recurrence algorithm, which is numerically stable: + /// + /// I_{n-1}(x) = (2n/x) I_n(x) + I_{n+1}(x) + /// + /// The result is normalized against the known value of I₀(x). + /// + /// + /// Thrown if n < 0. + public static double In(int n, double x) + { + if (n < 0 || n > 1_000_000) + throw new ArgumentOutOfRangeException(nameof(n), "The order n must be between 0 and 1,000,000."); + if (n == 0) return I0(x); + if (n == 1) return I1(x); + if (x == 0.0) return 0.0; + + double ax = Math.Abs(x); + double tox = 2.0 / ax; + double bip = 0.0, bi = 1.0; + double ans = 0.0; + + // Starting index for downward recurrence + int m = 2 * (n + (int)Math.Sqrt(IACC * n)); + + for (int j = m; j > 0; j--) + { + double bim = bip + j * tox * bi; + bip = bi; + bi = bim; + + // Rescale to prevent overflow + if (Math.Abs(bi) > BIGNO) + { + ans *= BIGNI; + bi *= BIGNI; + bip *= BIGNI; + } + + if (j == n) ans = bip; + } + + // Normalize using I0 + ans *= I0(ax) / bi; + return (x < 0.0 && (n & 1) != 0) ? -ans : ans; + } + + #endregion + + #region Modified Bessel Functions of the Second Kind (K) + + /// + /// Computes the modified Bessel function of the second kind, order 0: K₀(x). + /// + /// The argument at which to evaluate K₀. Must be positive (x > 0). + /// The value of K₀(x). Always positive, monotonically decreasing. + /// + /// + /// Uses polynomial approximations from Abramowitz and Stegun (1972), sections 9.8.5 and 9.8.6. + /// For 0 < x <= 2, uses: K₀(x) = -ln(x/2)I₀(x) + polynomial in (x/2)², with |e| < 1x10⁻⁸. + /// For x > 2, uses: K₀(x) = (1/sqrt(x))exp(-x) * polynomial in (2/x), with |e| < 1.9x10⁻⁷. + /// + /// + /// K₀(x) decays exponentially as x → ∞ and diverges logarithmically as x → 0⁺. + /// + /// + /// Thrown if x <= 0. + public static double K0(double x) + { + if (x <= 0.0) + throw new ArgumentOutOfRangeException(nameof(x), "The argument x must be positive for K0."); + + if (x <= 2.0) + { + double t = x * x / 4.0; // (x/2)^2 + return -Math.Log(x / 2.0) * I0(x) + + (-0.57721566 + t * (0.42278420 + t * (0.23069756 + + t * (0.03488590 + t * (0.00262698 + t * (0.00010750 + + t * 0.00000740)))))); + } + else + { + double t = 2.0 / x; + return (Math.Exp(-x) / Math.Sqrt(x)) * + (1.25331414 + t * (-0.07832358 + t * (0.02189568 + + t * (-0.01062446 + t * (0.00587872 + t * (-0.00251540 + + t * 0.00053208)))))); + } + } + + /// + /// Computes the modified Bessel function of the second kind, order 1: K₁(x). + /// + /// The argument at which to evaluate K₁. Must be positive (x > 0). + /// The value of K₁(x). Always positive, monotonically decreasing. + /// + /// + /// Uses polynomial approximations from Abramowitz and Stegun (1972), sections 9.8.7 and 9.8.8. + /// For 0 < x <= 2, uses: x*K₁(x) = x*ln(x/2)*I₁(x) + polynomial in (x/2)², with |e| < 8x10⁻⁹. + /// For x > 2, uses: K₁(x) = (1/sqrt(x))exp(-x) * polynomial in (2/x), with |e| < 2.2x10⁻⁷. + /// + /// + /// K₁(x) decays exponentially as x → ∞ and diverges as 1/x as x → 0⁺. + /// + /// + /// Thrown if x <= 0. + public static double K1(double x) + { + if (x <= 0.0) + throw new ArgumentOutOfRangeException(nameof(x), "The argument x must be positive for K1."); + + if (x <= 2.0) + { + double t = x * x / 4.0; // (x/2)^2 + return Math.Log(x / 2.0) * I1(x) + (1.0 / x) * + (1.0 + t * (0.15443144 + t * (-0.67278579 + + t * (-0.18156897 + t * (-0.01919402 + t * (-0.00110404 + + t * (-0.00004686))))))); + } + else + { + double t = 2.0 / x; + return (Math.Exp(-x) / Math.Sqrt(x)) * + (1.25331414 + t * (0.23498619 + t * (-0.03655620 + + t * (0.01504268 + t * (-0.00780353 + t * (0.00325614 + + t * (-0.00068245))))))); + } + } + + /// + /// Computes the modified Bessel function of the second kind for integer order n: Kₙ(x). + /// + /// The order of the function. Must be non-negative (n >= 0). + /// The argument at which to evaluate Kₙ. Must be positive (x > 0). + /// The value of Kₙ(x). Always positive for x > 0. + /// + /// + /// For n = 0 and n = 1, delegates to and respectively. + /// For n >= 2, uses the forward recurrence relation, which is numerically stable for K: + /// + /// K_{n+1}(x) = K_{n-1}(x) + (2n/x) K_n(x) + /// + /// + /// + /// Thrown if n < 0 or x <= 0. + public static double Kn(int n, double x) + { + if (n < 0) + throw new ArgumentOutOfRangeException(nameof(n), "The order n must be non-negative."); + if (x <= 0.0) + throw new ArgumentOutOfRangeException(nameof(x), "The argument x must be positive for Kn."); + if (n == 0) return K0(x); + if (n == 1) return K1(x); + + double tox = 2.0 / x; + double bkm = K0(x); + double bk = K1(x); + + for (int j = 1; j < n; j++) + { + double bkp = bkm + j * tox * bk; + bkm = bk; + bk = bkp; + } + + return bk; + } + + #endregion + + #region Bessel Functions of the First Kind (J) + + /// + /// Computes the Bessel function of the first kind, order 0: J₀(x). + /// + /// The argument at which to evaluate J₀. Can be any real number. + /// The value of J₀(x). J₀(0) = 1. J₀(x) = J₀(-x) (even function). Range: [-0.4028, 1]. + /// + /// + /// Uses rational polynomial approximations for |x| < 8 and Hankel's asymptotic expansion + /// for |x| >= 8, following Press et al. (2007). Accuracy is approximately double precision + /// for |x| < 8 and approximately 10⁻⁸ for the asymptotic region. + /// + /// + /// J₀(x) is the unique solution of Bessel's equation of order 0 that is finite at x = 0. + /// It oscillates with decreasing amplitude: J₀(x) ~ sqrt(2/(pi*x)) cos(x - pi/4) for large x. + /// + /// + public static double J0(double x) + { + double ax = Math.Abs(x); + if (ax < 8.0) + { + double y = x * x; + double ans1 = 57568490574.0 + y * (-13362590354.0 + y * (651619640.7 + + y * (-11214424.18 + y * (77392.33017 + y * (-184.9052456))))); + double ans2 = 57568490411.0 + y * (1029532985.0 + y * (9494680.718 + + y * (59272.64853 + y * (267.8532712 + y * 1.0)))); + return ans1 / ans2; + } + else + { + double z = 8.0 / ax; + double y = z * z; + double xx = ax - 0.785398164; // ax - pi/4 + double p = 1.0 + y * (-0.1098628627e-2 + y * (0.2734510407e-4 + + y * (-0.2073370639e-5 + y * 0.2093887211e-6))); + double q = -0.1562499995e-1 + y * (0.1430488765e-3 + + y * (-0.6911147651e-5 + y * (0.7621095161e-6 + + y * (-0.934945152e-7)))); + return Math.Sqrt(0.636619772 / ax) * (Math.Cos(xx) * p - z * Math.Sin(xx) * q); + } + } + + /// + /// Computes the Bessel function of the first kind, order 1: J₁(x). + /// + /// The argument at which to evaluate J₁. Can be any real number. + /// The value of J₁(x). J₁(0) = 0. J₁(-x) = -J₁(x) (odd function). Range: [-0.5819, 0.5819]. + /// + /// + /// Uses rational polynomial approximations for |x| < 8 and Hankel's asymptotic expansion + /// for |x| >= 8, following Press et al. (2007). Accuracy is approximately double precision + /// for |x| < 8 and approximately 10⁻⁸ for the asymptotic region. + /// + /// + /// J₁(x) oscillates with decreasing amplitude: J₁(x) ~ sqrt(2/(pi*x)) cos(x - 3*pi/4) for large x. + /// + /// + public static double J1(double x) + { + double ax = Math.Abs(x); + double ans; + if (ax < 8.0) + { + double y = x * x; + double ans1 = x * (72362614232.0 + y * (-7895059235.0 + y * (242396853.1 + + y * (-2972611.439 + y * (15704.48260 + y * (-30.16036606)))))); + double ans2 = 144725228442.0 + y * (2300535178.0 + y * (18583304.74 + + y * (99447.43394 + y * (376.9991397 + y * 1.0)))); + ans = ans1 / ans2; + } + else + { + double z = 8.0 / ax; + double y = z * z; + double xx = ax - 2.356194491; // ax - 3*pi/4 + double p = 1.0 + y * (0.183105e-2 + y * (-0.3516396496e-4 + + y * (0.2457520174e-5 + y * (-0.240337019e-6)))); + double q = 0.04687499995 + y * (-0.2002690873e-3 + + y * (0.8449199096e-5 + y * (-0.88228987e-6 + + y * 0.105787412e-6))); + ans = Math.Sqrt(0.636619772 / ax) * (Math.Cos(xx) * p - z * Math.Sin(xx) * q); + if (x < 0.0) ans = -ans; + } + return ans; + } + + /// + /// Computes the Bessel function of the first kind for integer order n: Jₙ(x). + /// + /// The order of the function. Must be non-negative (n >= 0). + /// The argument at which to evaluate Jₙ. Can be any real number. + /// + /// The value of Jₙ(x). For odd n, Jₙ(-x) = -Jₙ(x). For even n, Jₙ(-x) = Jₙ(x). + /// Jₙ(0) = 0 for n > 0, and J₀(0) = 1. + /// + /// + /// + /// For n = 0 and n = 1, delegates to and respectively. + /// For n >= 2, uses two strategies depending on the relative magnitudes of n and |x|: + /// + /// + /// When |x| > n, forward recurrence from J₀ and J₁ is numerically stable: + /// + /// J_{n+1}(x) = (2n/x) J_n(x) - J_{n-1}(x) + /// + /// + /// + /// When |x| <= n, Miller's downward recurrence is used, starting from a large order m >> n + /// and recursing down. The result is normalized using the identity: + /// + /// J₀(x) + 2 J₂(x) + 2 J₄(x) + ... = 1 + /// + /// + /// + /// Thrown if n < 0. + public static double Jn(int n, double x) + { + if (n < 0 || n > 1_000_000) + throw new ArgumentOutOfRangeException(nameof(n), "The order n must be between 0 and 1,000,000."); + if (n == 0) return J0(x); + if (n == 1) return J1(x); + + double ax = Math.Abs(x); + if (ax == 0.0) return 0.0; + + double ans; + if (ax > (double)n) + { + // Forward recurrence from J0, J1 (stable for x > n) + double tox = 2.0 / ax; + double bjm = J0(ax); + double bj = J1(ax); + for (int j = 1; j < n; j++) + { + double bjp = j * tox * bj - bjm; + bjm = bj; + bj = bjp; + } + ans = bj; + } + else + { + // Miller's downward recurrence (stable for x <= n) + double tox = 2.0 / ax; + + // Starting index - must be even and much larger than n + int m = 2 * ((n + (int)Math.Sqrt(IACC * n)) / 2); + bool jsum = false; + double bjp = 0.0; + ans = 0.0; + double sum = 0.0; + double bj = 1.0; + + for (int j = m; j > 0; j--) + { + double bjm = j * tox * bj - bjp; + bjp = bj; + bj = bjm; + + // Rescale to prevent overflow + if (Math.Abs(bj) > BIGNO) + { + bj *= BIGNI; + bjp *= BIGNI; + ans *= BIGNI; + sum *= BIGNI; + } + + if (jsum) sum += bj; + jsum = !jsum; + if (j == n) ans = bjp; + } + + // Normalize: J0 + 2*J2 + 2*J4 + ... = 1 + sum = 2.0 * sum - bj; + ans /= sum; + } + + return (x < 0.0 && (n & 1) != 0) ? -ans : ans; + } + + #endregion + + #region Bessel Functions of the Second Kind (Y) + + /// + /// Computes the Bessel function of the second kind, order 0: Y₀(x). + /// + /// The argument at which to evaluate Y₀. Must be positive (x > 0). + /// The value of Y₀(x). Y₀(x) → -∞ as x → 0⁺. Oscillates for large x. + /// + /// + /// Uses rational polynomial approximations for x < 8 and Hankel's asymptotic expansion + /// for x >= 8, following Press et al. (2007). For x < 8, the approximation involves + /// J₀(x)·ln(x), reflecting the logarithmic singularity at the origin. + /// + /// + /// Y₀(x) is the second linearly independent solution of Bessel's equation of order 0. + /// Unlike J₀, it is singular at x = 0. For large x: Y₀(x) ~ sqrt(2/(pi*x)) sin(x - pi/4). + /// + /// + /// Thrown if x <= 0. + public static double Y0(double x) + { + if (x <= 0.0) + throw new ArgumentOutOfRangeException(nameof(x), "The argument x must be positive for Y0."); + + if (x < 8.0) + { + double y = x * x; + double ans1 = -2957821389.0 + y * (7062834065.0 + y * (-512359803.6 + + y * (10879881.29 + y * (-86327.92757 + y * 228.4622733)))); + double ans2 = 40076544269.0 + y * (745249964.8 + y * (7189466.438 + + y * (47447.26470 + y * (226.1030244 + y * 1.0)))); + return (ans1 / ans2) + 0.636619772 * J0(x) * Math.Log(x); + } + else + { + double z = 8.0 / x; + double y = z * z; + double xx = x - 0.785398164; // x - pi/4 + double p = 1.0 + y * (-0.1098628627e-2 + y * (0.2734510407e-4 + + y * (-0.2073370639e-5 + y * 0.2093887211e-6))); + double q = -0.1562499995e-1 + y * (0.1430488765e-3 + + y * (-0.6911147651e-5 + y * (0.7621095161e-6 + + y * (-0.934945152e-7)))); + return Math.Sqrt(0.636619772 / x) * (Math.Sin(xx) * p + z * Math.Cos(xx) * q); + } + } + + /// + /// Computes the Bessel function of the second kind, order 1: Y₁(x). + /// + /// The argument at which to evaluate Y₁. Must be positive (x > 0). + /// The value of Y₁(x). Y₁(x) → -∞ as x → 0⁺. Oscillates for large x. + /// + /// + /// Uses rational polynomial approximations for x < 8 and Hankel's asymptotic expansion + /// for x >= 8, following Press et al. (2007). For x < 8, the approximation involves + /// J₁(x)·ln(x) - 1/x, reflecting the singular behavior at the origin. + /// + /// + /// For large x: Y₁(x) ~ sqrt(2/(pi*x)) sin(x - 3*pi/4). + /// + /// + /// Thrown if x <= 0. + public static double Y1(double x) + { + if (x <= 0.0) + throw new ArgumentOutOfRangeException(nameof(x), "The argument x must be positive for Y1."); + + if (x < 8.0) + { + double y = x * x; + double ans1 = x * (-4900604943000.0 + y * (1275274390000.0 + + y * (-51534381390.0 + y * (734926455.1 + + y * (-4237922.726 + y * 8511.937935))))); + double ans2 = 24995805700000.0 + y * (424441966400.0 + + y * (3733650367.0 + y * (22459040.02 + + y * (102042.6050 + y * (354.9632885 + y * 1.0))))); + return (ans1 / ans2) + 0.636619772 * (J1(x) * Math.Log(x) - 1.0 / x); + } + else + { + double z = 8.0 / x; + double y = z * z; + double xx = x - 2.356194491; // x - 3*pi/4 + double p = 1.0 + y * (0.183105e-2 + y * (-0.3516396496e-4 + + y * (0.2457520174e-5 + y * (-0.240337019e-6)))); + double q = 0.04687499995 + y * (-0.2002690873e-3 + + y * (0.8449199096e-5 + y * (-0.88228987e-6 + + y * 0.105787412e-6))); + return Math.Sqrt(0.636619772 / x) * (Math.Sin(xx) * p + z * Math.Cos(xx) * q); + } + } + + /// + /// Computes the Bessel function of the second kind for integer order n: Yₙ(x). + /// + /// The order of the function. Must be non-negative (n >= 0). + /// The argument at which to evaluate Yₙ. Must be positive (x > 0). + /// The value of Yₙ(x). Singular at x = 0. + /// + /// + /// For n = 0 and n = 1, delegates to and respectively. + /// For n >= 2, uses the forward recurrence relation, which is numerically stable for Y: + /// + /// Y_{n+1}(x) = (2n/x) Y_n(x) - Y_{n-1}(x) + /// + /// + /// + /// Thrown if n < 0 or x <= 0. + public static double Yn(int n, double x) + { + if (n < 0) + throw new ArgumentOutOfRangeException(nameof(n), "The order n must be non-negative."); + if (x <= 0.0) + throw new ArgumentOutOfRangeException(nameof(x), "The argument x must be positive for Yn."); + if (n == 0) return Y0(x); + if (n == 1) return Y1(x); + + double tox = 2.0 / x; + double bym = Y0(x); + double by = Y1(x); + + for (int j = 1; j < n; j++) + { + double byp = j * tox * by - bym; + bym = by; + by = byp; + } + + return by; + } + + #endregion + + } +} diff --git a/Numerics/Mathematics/Special Functions/Gamma.cs b/Numerics/Mathematics/Special Functions/Gamma.cs index 8d045001..61dca2bf 100644 --- a/Numerics/Mathematics/Special Functions/Gamma.cs +++ b/Numerics/Mathematics/Special Functions/Gamma.cs @@ -50,7 +50,7 @@ // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA using System; -using Microsoft.VisualBasic; + using Numerics.Distributions; namespace Numerics.Mathematics.SpecialFunctions diff --git a/Numerics/Numerics.csproj b/Numerics/Numerics.csproj index 380eca32..3d25aa0a 100644 --- a/Numerics/Numerics.csproj +++ b/Numerics/Numerics.csproj @@ -1,7 +1,7 @@  - net9.0;net8.0;net481 + net10.0;net9.0;net8.0;net481 enable True true @@ -27,8 +27,8 @@ - - CS1591;CS1587 + + CS1587 2.0.0 This major update delivers broad improvements across the entire Numerics library, with substantial enhancements to data analysis, statistical modeling, numerical methods, distributions, optimization, machine learning utilities, and overall performance and reliability. The update reflects over a year of incremental development, code modernization, and refinement. 2.0.0.0 @@ -36,6 +36,7 @@ enable + true diff --git a/Numerics/Sampling/Bootstrap/Bootstrap.cs b/Numerics/Sampling/Bootstrap/Bootstrap.cs index c64db1c7..eaca193b 100644 --- a/Numerics/Sampling/Bootstrap/Bootstrap.cs +++ b/Numerics/Sampling/Bootstrap/Bootstrap.cs @@ -1,4 +1,4 @@ -/* +/* * NOTICE: * The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about * the results, or appropriateness of outputs, obtained from Numerics. @@ -28,59 +28,126 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +using Numerics.Data.Statistics; +using Numerics.Distributions; using Numerics.Mathematics.Optimization; using System; -using System.Collections.Generic; -using System.Diagnostics; using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; namespace Numerics.Sampling { + + /// + /// A general-purpose bootstrap class for parametric or non-parametric bootstrap analysis. + /// Supports all major confidence interval methods: Percentile, Bias-Corrected, BCa, Normal, and Bootstrap-t. + /// + /// The type of data being bootstrapped. + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + /// + /// public class Bootstrap { + #region Construction + + /// + /// Constructs a new bootstrap analysis. + /// + /// The original data. + /// The original fitted parameter set. + public Bootstrap(TData originalData, ParameterSet originalParameters) + { + _originalData = originalData; + _originalParameters = originalParameters; + } + + #endregion + + #region Members + + private TData _originalData; + private ParameterSet _originalParameters; + private ParameterSet[] _bootstrapParameterSets = null!; + private double[,] _bootstrapStatistics = null!; + private int _numStats; + private int _numParams; + private int _failedCount; + private bool[] _validFlags = null!; + private double[,]? _studentizedValues; + private double[,]? _transformedStatistics; + + #endregion + + #region Properties + + /// + /// Delegate function for resampling the original data given the current parameters and a random number generator. + /// + public Func? ResampleFunction { get; set; } + + /// + /// Delegate function for fitting a model to data and returning a parameter set. + /// + public Func? FitFunction { get; set; } + + /// + /// Delegate function for extracting statistics from a fitted parameter set. + /// Returns an array of statistic values (e.g., quantiles at multiple probabilities). + /// + public Func? StatisticFunction { get; set; } + /// - /// Delegate function for resampling the original data and model fit. + /// Optional delegate for computing a leave-one-out jackknife sample. + /// Takes the original data and the index of the observation to remove. + /// Required for the BCa confidence interval method. /// - public Func ResampleFunction { get; set; } = null!; + public Func? JackknifeFunction { get; set; } /// - /// Delegate function for fitting a model. + /// Optional delegate that returns the number of observations in the data. + /// Required for the BCa confidence interval method. /// - public Func FitFunction { get; set; } = null!; + public Func? SampleSizeFunction { get; set; } /// - /// Delegate function for extracting a statistic from the fit result. + /// Optional transform applied to statistic values before Normal and Bootstrap-t CI computation. + /// Default is cube-root: x → x^(1/3). Set to null for no transform. /// - public Func StatisticFunction { get; set; } = null!; + public Func Transform { get; set; } = x => Math.Pow(x, 1d / 3d); /// - /// Number of bootstrap replicates. + /// Optional inverse transform corresponding to Transform. + /// Default is cube: x → x^3. Set to null for no transform. + /// + public Func InverseTransform { get; set; } = x => Math.Pow(x, 3d); + + /// + /// Number of bootstrap replicates. Default = 10,000. /// public int Replicates { get; set; } = 10000; /// - /// Gets and sets the PRNG seed for reproducibility. + /// Gets and sets the PRNG seed for reproducibility. Default = 12345. /// public int PRNGSeed { get; set; } = 12345; /// - /// Constructs a new bootstrap class. + /// The maximum number of retries for a failed bootstrap replicate. Default = 20. /// - /// The original data. - /// The original fitted parameter set. - public Bootstrap(TData originalData, ParameterSet originalParameters) - { - _originalData = originalData; - _originalParameters = originalParameters; - } + public int MaxRetries { get; set; } = 20; - private TData _originalData; - private ParameterSet _originalParameters; - private ParameterSet[] _bootstrapParameterSets = null!; - private double[][] _bootstrapStatistics = null!; + /// + /// The number of inner bootstrap replicates for Bootstrap-t standard error estimation. Default = 300. + /// + public int InnerReplicates { get; set; } = 300; /// /// Gets the bootstrapped model parameter sets. @@ -88,129 +155,664 @@ public Bootstrap(TData originalData, ParameterSet originalParameters) public ParameterSet[] BootstrapParameterSets => _bootstrapParameterSets; /// - /// Gets the bootstrapped statistics. + /// Gets the bootstrapped statistics as a 2D array [replicates, statistics]. + /// + public double[,] BootstrapStatistics => _bootstrapStatistics; + + /// + /// Gets the number of replicates that failed after all retries. /// - public IReadOnlyList BootstrapStatistics => _bootstrapStatistics; + public int FailedReplicates => _failedCount; + + #endregion + + #region Run Methods /// - /// Runs the basic bootstrap procedure. + /// Runs the basic bootstrap procedure with error handling and retry logic. /// public void Run() { - if (ResampleFunction == null) - throw new InvalidOperationException("Bootstrap Sample Function must be set."); - if (FitFunction == null) - throw new InvalidOperationException("Fit Function must be set."); - if (StatisticFunction == null) - throw new InvalidOperationException("Statistic Function must be set."); - - _bootstrapParameterSets = new ParameterSet[Replicates]; - _bootstrapStatistics = new double[Replicates][]; + ValidateCoreDelegates(); + var resample = ResampleFunction!; + var fit = FitFunction!; + var statistic = StatisticFunction!; + InitializeState(); var prng = new MersenneTwister(PRNGSeed); var seeds = prng.NextIntegers(Replicates); Parallel.For(0, Replicates, idx => { - var rng = new MersenneTwister(seeds[idx]); - var sample = ResampleFunction(_originalData, _originalParameters, rng); - var fit = FitFunction(sample); - var stat = StatisticFunction(fit); - _bootstrapParameterSets[idx] = fit; - _bootstrapStatistics[idx] = stat; + bool succeeded = false; + for (int retry = 0; retry < MaxRetries; retry++) + { + try + { + var rng = new MersenneTwister(seeds[idx] + 10 * retry); + var sample = resample(_originalData, _originalParameters, rng); + var fitResult = fit(sample); + var stat = statistic(fitResult); + + // Validate: check for NaN in statistics + bool hasNaN = false; + for (int k = 0; k < _numStats; k++) + { + if (double.IsNaN(stat[k]) || double.IsInfinity(stat[k])) + { + hasNaN = true; + break; + } + } + if (hasNaN) continue; + + _bootstrapParameterSets[idx] = fitResult; + for (int k = 0; k < _numStats; k++) + _bootstrapStatistics[idx, k] = stat[k]; + _validFlags[idx] = true; + succeeded = true; + } + catch (Exception) + { + // retry + } + if (succeeded) break; + } + + if (!succeeded) + { + MarkFailed(idx); + } }); } + /// + /// Runs the double bootstrap procedure with bias correction. + /// + /// Number of inner bootstrap replicates. Default = 300. public void RunDoubleBootstrap(int innerReplicates = 300) { - if (ResampleFunction == null) - throw new InvalidOperationException("Bootstrap Sample Function must be set."); - if (FitFunction == null) - throw new InvalidOperationException("Fit Function must be set."); - if (StatisticFunction == null) - throw new InvalidOperationException("Statistic Function must be set."); + ValidateCoreDelegates(); + var resample = ResampleFunction!; + var fit = FitFunction!; + var statistic = StatisticFunction!; + InitializeState(); + + var prng = new MersenneTwister(PRNGSeed); + var seeds = prng.NextIntegers(Replicates); + + Parallel.For(0, Replicates, idx => + { + bool succeeded = false; + for (int retry = 0; retry < MaxRetries; retry++) + { + try + { + var rng = new MersenneTwister(seeds[idx] + 10 * retry); + + // Outer bootstrap + var outerSample = resample(_originalData, _originalParameters, rng); + var outerFit = fit(outerSample); + var outerStat = statistic(outerFit); + + // Inner bootstrap for bias estimation + int p = outerFit.Values.Length; + var parmsInnerSum = new double[p]; + var statsInnerSum = new double[_numStats]; + int validInner = 0; + + for (int k = 0; k < innerReplicates; k++) + { + try + { + var innerSample = resample(outerSample, outerFit, rng); + var innerFit = fit(innerSample); + var innerStat = statistic(innerFit); + + for (int i = 0; i < p; i++) + parmsInnerSum[i] += innerFit.Values[i]; + for (int i = 0; i < _numStats; i++) + statsInnerSum[i] += innerStat[i]; + validInner++; + } + catch (Exception) + { + // skip failed inner replicate + } + } + + if (validInner == 0) continue; + + // Bias-correct parameters + var biasCorrectedParms = new double[p]; + for (int i = 0; i < p; i++) + { + double innerMean = parmsInnerSum[i] / validInner; + biasCorrectedParms[i] = outerFit.Values[i] - (innerMean - outerFit.Values[i]); + } + + // Bias-correct statistics + var biasCorrectedStats = new double[_numStats]; + for (int i = 0; i < _numStats; i++) + { + double innerMean = statsInnerSum[i] / validInner; + biasCorrectedStats[i] = outerStat[i] - (innerMean - outerStat[i]); + } + + _bootstrapParameterSets[idx] = new ParameterSet(biasCorrectedParms, outerFit.Fitness); + for (int k = 0; k < _numStats; k++) + _bootstrapStatistics[idx, k] = biasCorrectedStats[k]; + _validFlags[idx] = true; + succeeded = true; + } + catch (Exception) + { + // retry + } + if (succeeded) break; + } + + if (!succeeded) + { + MarkFailed(idx); + } + }); + } + + /// + /// Runs the bootstrap procedure with nested inner bootstrap for studentized (Bootstrap-t) confidence intervals. + /// Must be called before requesting Bootstrap-t CIs. + /// + public void RunWithStudentizedBootstrap() + { + ValidateCoreDelegates(); + var resample = ResampleFunction!; + var fitFunc = FitFunction!; + var statistic = StatisticFunction!; + + var originalStats = statistic(_originalParameters); + _numStats = originalStats.Length; + _numParams = _originalParameters.Values.Length; + + // Apply transform to population statistics + var popTransformed = new double[_numStats]; + for (int i = 0; i < _numStats; i++) + popTransformed[i] = ApplyTransform(originalStats[i]); _bootstrapParameterSets = new ParameterSet[Replicates]; - _bootstrapStatistics = new double[Replicates][]; + _bootstrapStatistics = new double[Replicates, _numStats]; + _validFlags = new bool[Replicates]; + _failedCount = 0; + _studentizedValues = new double[Replicates, _numStats]; + _transformedStatistics = new double[Replicates, _numStats]; + var studentizedValues = _studentizedValues; + var transformedStatistics = _transformedStatistics; var prng = new MersenneTwister(PRNGSeed); var seeds = prng.NextIntegers(Replicates); Parallel.For(0, Replicates, idx => { - var rng = new MersenneTwister(seeds[idx]); + bool succeeded = false; + for (int retry = 0; retry < MaxRetries; retry++) + { + try + { + var rng = new MersenneTwister(seeds[idx] + 10 * retry); + var sample = resample(_originalData, _originalParameters, rng); + var outerFit = fitFunc(sample); + var outerStats = statistic(outerFit); - // Step 1: outer bootstrap - var outerSample = ResampleFunction(_originalData, _originalParameters, rng); - var outerFit = FitFunction(outerSample); - var outerStat = StatisticFunction(outerFit); + _bootstrapParameterSets[idx] = outerFit; + for (int k = 0; k < _numStats; k++) + _bootstrapStatistics[idx, k] = outerStats[k]; - // Step 2: inner bootstrap - int p = outerFit.Values.Length; - var parmsInnerSum = new double[p]; - int s = outerStat.Length; - var statsInnerSum = new double[s]; + // Transform outer statistics + var outerTransformed = new double[_numStats]; + for (int j = 0; j < _numStats; j++) + outerTransformed[j] = ApplyTransform(outerStats[j]); - for (int k = 0; k < innerReplicates; k++) - { - var innerSample = ResampleFunction(outerSample, outerFit, rng); - var innerFit = FitFunction(innerSample); - var innerStat = StatisticFunction(innerFit); - - for (int i = 0; i < p; i++) - parmsInnerSum[i] += innerFit.Values[i]; - for (int i = 0; i < s; i++) - statsInnerSum[i] += innerStat[i]; + // Inner bootstrap for SE estimation + var innerPrng = new MersenneTwister(seeds[idx]); + var innerSeeds = innerPrng.NextIntegers(InnerReplicates); + var innerTransformed = new double[InnerReplicates, _numStats]; + int validInner = 0; + + for (int k = 0; k < InnerReplicates; k++) + { + try + { + var innerSample = resample(sample, outerFit, new MersenneTwister(innerSeeds[k])); + var innerFit = fitFunc(innerSample); + var innerStats = statistic(innerFit); + for (int j = 0; j < _numStats; j++) + innerTransformed[k, j] = ApplyTransform(innerStats[j]); + validInner++; + } + catch (Exception) + { + for (int j = 0; j < _numStats; j++) + innerTransformed[k, j] = double.NaN; + } + } + + if (validInner < 2) continue; + + // Compute inner SE per statistic and studentized values + for (int j = 0; j < _numStats; j++) + { + var col = innerTransformed.GetColumn(j); + var validCol = col.Where(x => !double.IsNaN(x) && !double.IsInfinity(x)).ToArray(); + double se = validCol.Length > 1 ? Statistics.StandardDeviation(validCol) : double.NaN; + transformedStatistics[idx, j] = outerTransformed[j]; + studentizedValues[idx, j] = se > 0 ? (popTransformed[j] - outerTransformed[j]) / se : double.NaN; + } + + _validFlags[idx] = true; + succeeded = true; + } + catch (Exception) + { + // retry + } + if (succeeded) break; } - // bias correct the parameters - var biasCorrectedParms = new double[p]; - for (int i = 0; i < p; i++) + if (!succeeded) { - double innerMean = parmsInnerSum[i] / innerReplicates; - biasCorrectedParms[i] = outerFit.Values[i] - (innerMean - outerFit.Values[i]); + MarkFailed(idx); + for (int j = 0; j < _numStats; j++) + { + transformedStatistics[idx, j] = double.NaN; + studentizedValues[idx, j] = double.NaN; + } } + }); + } - // bias correct the statistics - var biasCorrectedStats = new double[s]; - for (int i = 0; i < s; i++) + #endregion + + #region Confidence Intervals + + /// + /// Computes bootstrap confidence intervals using the specified method. + /// + /// The confidence interval method. + /// The confidence level. Default = 0.1, resulting in 90% CIs. + /// A BootstrapResults object containing CIs for both parameters and statistics. + public BootstrapResults GetConfidenceIntervals(BootstrapCIMethod method, double alpha = 0.1) + { + if (_bootstrapStatistics == null) + throw new InvalidOperationException("Run() or RunWithStudentizedBootstrap() must be called first."); + if (alpha <= 0 || alpha >= 1) + throw new ArgumentOutOfRangeException(nameof(alpha), "Alpha must be between 0 and 1."); + if (method == BootstrapCIMethod.BCa && (JackknifeFunction == null || SampleSizeFunction == null)) + throw new InvalidOperationException("JackknifeFunction and SampleSizeFunction must be set for BCa method."); + if (method == BootstrapCIMethod.BootstrapT && _studentizedValues == null) + throw new InvalidOperationException("RunWithStudentizedBootstrap() must be called before requesting Bootstrap-t CIs."); + + if (StatisticFunction == null) + throw new InvalidOperationException("StatisticFunction must be set."); + var originalStats = StatisticFunction(_originalParameters); + + // Compute acceleration constants once for BCa + double[]? accelConstants = null; + if (method == BootstrapCIMethod.BCa) + accelConstants = ComputeAccelerationConstants(originalStats); + + var results = new BootstrapResults + { + Method = method, + Alpha = alpha, + StatisticResults = new BootstrapStatisticResult[_numStats], + ParameterResults = new BootstrapStatisticResult[_numParams], + FailedReplicates = _failedCount + }; + + // Compute CIs for each statistic + for (int i = 0; i < _numStats; i++) + { + var values = _bootstrapStatistics.GetColumn(i); + switch (method) { - double innerMean = statsInnerSum[i] / innerReplicates; - biasCorrectedStats[i] = outerStat[i] - (innerMean - outerStat[i]); + case BootstrapCIMethod.Percentile: + results.StatisticResults[i] = ComputePercentileCI(values, originalStats[i], alpha); + break; + case BootstrapCIMethod.BiasCorrected: + results.StatisticResults[i] = ComputeBiasCorrectedCI(values, originalStats[i], alpha); + break; + case BootstrapCIMethod.BCa: + results.StatisticResults[i] = ComputeBCaCI(values, originalStats[i], alpha, accelConstants![i]); + break; + case BootstrapCIMethod.Normal: + results.StatisticResults[i] = ComputeNormalCI(values, originalStats[i], alpha); + break; + case BootstrapCIMethod.BootstrapT: + results.StatisticResults[i] = ComputeBootstrapTCI(i, originalStats[i], alpha); + break; } + } - _bootstrapParameterSets[idx].Values = biasCorrectedParms; - _bootstrapStatistics[idx] = biasCorrectedStats; - }); + // Compute percentile CIs for each parameter + for (int i = 0; i < _numParams; i++) + { + var values = _bootstrapParameterSets.Select(ps => ps.Values[i]).ToArray(); + results.ParameterResults[i] = ComputePercentileCI(values, _originalParameters.Values[i], alpha); + } + + return results; + } + + #endregion + + #region CI Methods + + /// + /// Computes percentile confidence intervals for a single statistic. + /// + private BootstrapStatisticResult ComputePercentileCI(double[] values, double populationEstimate, double alpha) + { + var validValues = values.Where(x => !double.IsNaN(x) && !double.IsInfinity(x)).ToArray(); + Array.Sort(validValues); + + double lowerP = alpha / 2d; + double upperP = 1d - alpha / 2d; + + return new BootstrapStatisticResult + { + PopulationEstimate = populationEstimate, + LowerCI = validValues.Length > 0 ? Statistics.Percentile(validValues, lowerP, true) : double.NaN, + UpperCI = validValues.Length > 0 ? Statistics.Percentile(validValues, upperP, true) : double.NaN, + ValidCount = validValues.Length, + TotalCount = values.Length, + StandardError = validValues.Length > 1 ? Statistics.StandardDeviation(validValues) : double.NaN, + Mean = validValues.Length > 0 ? Statistics.Mean(validValues) : double.NaN + }; } /// - /// Returns percentile confidence intervals for each parameter. + /// Computes bias-corrected (BC) confidence intervals for a single statistic. /// - /// Confidence level. Default = 0.9. - public (double[] Lower, double[] Upper) GetPercentileConfidenceIntervals(double confidenceLevel = 0.90) + private BootstrapStatisticResult ComputeBiasCorrectedCI(double[] values, double populationEstimate, double alpha) { - if (_bootstrapStatistics is null) - throw new InvalidOperationException("Run() must be called before requesting CIs."); + var validValues = values.Where(x => !double.IsNaN(x) && !double.IsInfinity(x)).ToArray(); + int validN = validValues.Length; + if (validN == 0) return EmptyResult(populationEstimate, values.Length); - int nParams = _bootstrapStatistics[0].Length; - int n = _bootstrapStatistics.Length; - var lower = new double[nParams]; - var upper = new double[nParams]; + // Count proportion <= population estimate + int countLeq = 0; + for (int i = 0; i < validN; i++) + if (validValues[i] <= populationEstimate) countLeq++; + double P0 = (double)countLeq / (validN + 1); - double alpha = (1.0 - confidenceLevel) / 2.0; - int iLo = Math.Max(0, (int)Math.Floor(alpha * n)); - int iHi = Math.Min(n - 1, (int)Math.Ceiling((1.0 - alpha) * n) - 1); + Array.Sort(validValues); - for (int j = 0; j < nParams; j++) + double Z0 = Normal.StandardZ(P0); + double ZLower = Normal.StandardZ(alpha / 2d); + double ZUpper = Normal.StandardZ(1d - alpha / 2d); + double bcLower = Normal.StandardCDF(2d * Z0 + ZLower); + double bcUpper = Normal.StandardCDF(2d * Z0 + ZUpper); + + return new BootstrapStatisticResult + { + PopulationEstimate = populationEstimate, + LowerCI = Statistics.Percentile(validValues, bcLower, true), + UpperCI = Statistics.Percentile(validValues, bcUpper, true), + ValidCount = validN, + TotalCount = values.Length, + StandardError = validN > 1 ? Statistics.StandardDeviation(validValues) : double.NaN, + Mean = Statistics.Mean(validValues) + }; + } + + /// + /// Computes bias-corrected and accelerated (BCa) confidence intervals for a single statistic. + /// + private BootstrapStatisticResult ComputeBCaCI(double[] values, double populationEstimate, double alpha, double acceleration) + { + var validValues = values.Where(x => !double.IsNaN(x) && !double.IsInfinity(x)).ToArray(); + int validN = validValues.Length; + if (validN == 0) return EmptyResult(populationEstimate, values.Length); + + // Count proportion <= population estimate (matching BCaQuantileCI line 593) + int countLeq = 0; + for (int i = 0; i < validN; i++) + if (validValues[i] <= populationEstimate) countLeq++; + double P0 = (double)(countLeq + 1) / (validN + 1); + + Array.Sort(validValues); + + double Z0 = Normal.StandardZ(P0); + double ZLower = Normal.StandardZ(alpha / 2d); + double ZUpper = Normal.StandardZ(1d - alpha / 2d); + + double numLower = Z0 + ZLower; + double denLower = 1d - acceleration * numLower; + double bcLower = Normal.StandardCDF(Z0 + numLower / denLower); + + double numUpper = Z0 + ZUpper; + double denUpper = 1d - acceleration * numUpper; + double bcUpper = Normal.StandardCDF(Z0 + numUpper / denUpper); + + return new BootstrapStatisticResult { - double[] values = _bootstrapStatistics.Select(s => s[j]).ToArray(); - Array.Sort(values); - lower[j] = values[iLo]; - upper[j] = values[iHi]; + PopulationEstimate = populationEstimate, + LowerCI = Statistics.Percentile(validValues, bcLower, true), + UpperCI = Statistics.Percentile(validValues, bcUpper, true), + ValidCount = validN, + TotalCount = values.Length, + StandardError = validN > 1 ? Statistics.StandardDeviation(validValues) : double.NaN, + Mean = Statistics.Mean(validValues) + }; + } + + /// + /// Computes Normal (standard) confidence intervals for a single statistic. + /// Uses a configurable transform for transformation invariance. + /// + private BootstrapStatisticResult ComputeNormalCI(double[] values, double populationEstimate, double alpha) + { + double popTransformed = ApplyTransform(populationEstimate); + + var transformedValid = new double[values.Length]; + int validCount = 0; + for (int i = 0; i < values.Length; i++) + { + if (!double.IsNaN(values[i]) && !double.IsInfinity(values[i])) + { + transformedValid[validCount] = ApplyTransform(values[i]); + validCount++; + } } - return (lower, upper); + if (validCount < 2) return EmptyResult(populationEstimate, values.Length); + + var validSlice = new double[validCount]; + Array.Copy(transformedValid, validSlice, validCount); + + double SE = Statistics.StandardDeviation(validSlice); + double ZLower = Normal.StandardZ(alpha / 2d); + double ZUpper = Normal.StandardZ(1d - alpha / 2d); + + double lowerTransformed = popTransformed + SE * ZLower; + double upperTransformed = popTransformed + SE * ZUpper; + + return new BootstrapStatisticResult + { + PopulationEstimate = populationEstimate, + LowerCI = ApplyInverseTransform(lowerTransformed), + UpperCI = ApplyInverseTransform(upperTransformed), + ValidCount = validCount, + TotalCount = values.Length, + StandardError = SE, + Mean = Statistics.Mean(validSlice) + }; + } + + /// + /// Computes Bootstrap-t (studentized) confidence intervals for a single statistic. + /// Requires RunWithStudentizedBootstrap() to have been called first. + /// + private BootstrapStatisticResult ComputeBootstrapTCI(int statisticIndex, double populationEstimate, double alpha) + { + double popTransformed = ApplyTransform(populationEstimate); + + // GetConfidenceIntervals() validates _studentizedValues is non-null before calling this method + var xCol = _transformedStatistics!.GetColumn(statisticIndex); + var tCol = _studentizedValues!.GetColumn(statisticIndex); + + var validX = xCol.Where(x => !double.IsNaN(x) && !double.IsInfinity(x)).ToArray(); + var validT = tCol.Where(x => !double.IsNaN(x) && !double.IsInfinity(x)).ToArray(); + + if (validT.Length < 2) return EmptyResult(populationEstimate, Replicates); + + double SE = Statistics.StandardDeviation(validX); + Array.Sort(validT); + + double tLower = Statistics.Percentile(validT, alpha / 2d, true); + double tUpper = Statistics.Percentile(validT, 1d - alpha / 2d, true); + + return new BootstrapStatisticResult + { + PopulationEstimate = populationEstimate, + LowerCI = ApplyInverseTransform(popTransformed + SE * tLower), + UpperCI = ApplyInverseTransform(popTransformed + SE * tUpper), + ValidCount = validT.Length, + TotalCount = Replicates, + StandardError = SE, + Mean = Statistics.Mean(validX) + }; + } + + #endregion + + #region BCa Support + + /// + /// Computes the acceleration constants for each statistic using jackknife leave-one-out. + /// + private double[] ComputeAccelerationConstants(double[] populationEstimates) + { + // Caller (GetConfidenceIntervals) validates these delegates before calling this method + var sampleSize = SampleSizeFunction!; + var jackknife = JackknifeFunction!; + var fitFunc = FitFunction!; + var statistic = StatisticFunction!; + + int N = sampleSize(_originalData); + var I2 = new double[_numStats]; + var I3 = new double[_numStats]; + var a = new double[_numStats]; + + Parallel.For(0, N, idx => + { + try + { + var jackData = jackknife(_originalData, idx); + var jackFit = fitFunc(jackData); + var jackStats = statistic(jackFit); + + for (int i = 0; i < _numStats; i++) + { + double diff = populationEstimates[i] - jackStats[i]; + Tools.ParallelAdd(ref I2[i], diff * diff); + Tools.ParallelAdd(ref I3[i], diff * diff * diff); + } + } + catch (Exception) + { + // Skip failed jackknife samples + } + }); + + for (int i = 0; i < _numStats; i++) + a[i] = I3[i] / (Math.Pow(I2[i], 1.5) * 6d); + + return a; + } + + #endregion + + #region Private Helpers + + /// + /// Validates that the core delegate functions are set. + /// + private void ValidateCoreDelegates() + { + if (ResampleFunction == null) + throw new InvalidOperationException("ResampleFunction must be set."); + if (FitFunction == null) + throw new InvalidOperationException("FitFunction must be set."); + if (StatisticFunction == null) + throw new InvalidOperationException("StatisticFunction must be set."); } + + /// + /// Initializes internal state arrays before a bootstrap run. + /// + private void InitializeState() + { + // ValidateCoreDelegates() is always called before this method + var originalStats = StatisticFunction!(_originalParameters); + _numStats = originalStats.Length; + _numParams = _originalParameters.Values.Length; + + _bootstrapParameterSets = new ParameterSet[Replicates]; + _bootstrapStatistics = new double[Replicates, _numStats]; + _validFlags = new bool[Replicates]; + _failedCount = 0; + _studentizedValues = null; + _transformedStatistics = null; + } + + /// + /// Marks a replicate as failed with NaN values. + /// + private void MarkFailed(int idx) + { + var nanParams = new double[_numParams]; + for (int k = 0; k < _numParams; k++) nanParams[k] = double.NaN; + _bootstrapParameterSets[idx] = new ParameterSet(nanParams, double.NaN); + for (int k = 0; k < _numStats; k++) + _bootstrapStatistics[idx, k] = double.NaN; + _validFlags[idx] = false; + Interlocked.Increment(ref _failedCount); + } + + /// + /// Applies the Transform function, or returns the value unchanged if Transform is null. + /// + private double ApplyTransform(double value) + { + return Transform != null ? Transform(value) : value; + } + + /// + /// Applies the InverseTransform function, or returns the value unchanged if InverseTransform is null. + /// + private double ApplyInverseTransform(double value) + { + return InverseTransform != null ? InverseTransform(value) : value; + } + + /// + /// Creates an empty result for when there are insufficient valid values. + /// + private BootstrapStatisticResult EmptyResult(double populationEstimate, int totalCount) + { + return new BootstrapStatisticResult + { + PopulationEstimate = populationEstimate, + LowerCI = double.NaN, + UpperCI = double.NaN, + ValidCount = 0, + TotalCount = totalCount, + StandardError = double.NaN, + Mean = double.NaN + }; + } + + #endregion } } diff --git a/Numerics/Sampling/Bootstrap/BootstrapResults.cs b/Numerics/Sampling/Bootstrap/BootstrapResults.cs new file mode 100644 index 00000000..82eb1619 --- /dev/null +++ b/Numerics/Sampling/Bootstrap/BootstrapResults.cs @@ -0,0 +1,140 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; + +namespace Numerics.Sampling +{ + + /// + /// Enumeration of bootstrap confidence interval methods. + /// + public enum BootstrapCIMethod + { + /// + /// Percentile method. Direct percentile extraction from sorted bootstrap statistics. + /// + Percentile, + + /// + /// Bias-corrected (BC) percentile method. Adjusts for median bias using the standard normal transformation. + /// + BiasCorrected, + + /// + /// Bias-corrected and accelerated (BCa) method. Adjusts for both bias and skewness using jackknife acceleration. + /// + BCa, + + /// + /// Normal (standard) method. Uses a configurable transform for transformation invariance with normal approximation. + /// + Normal, + + /// + /// Bootstrap-t (studentized) method. Uses nested bootstrap to estimate standard errors and studentized pivotal statistic. + /// + BootstrapT + } + + /// + /// Stores bootstrap confidence interval results for a single statistic or parameter. + /// + [Serializable] + public class BootstrapStatisticResult + { + /// + /// The population (original) estimate for this statistic. + /// + public double PopulationEstimate { get; set; } + + /// + /// The lower confidence interval bound. + /// + public double LowerCI { get; set; } + + /// + /// The upper confidence interval bound. + /// + public double UpperCI { get; set; } + + /// + /// The number of valid (non-NaN) bootstrap replicates used. + /// + public int ValidCount { get; set; } + + /// + /// The total number of bootstrap replicates attempted. + /// + public int TotalCount { get; set; } + + /// + /// The bootstrap standard error (standard deviation of valid replicates). + /// + public double StandardError { get; set; } + + /// + /// The bootstrap mean of the valid replicates. + /// + public double Mean { get; set; } + } + + /// + /// Stores complete bootstrap analysis results including confidence intervals for statistics and parameters. + /// + [Serializable] + public class BootstrapResults + { + /// + /// The confidence interval method used. + /// + public BootstrapCIMethod Method { get; set; } + + /// + /// The alpha level used (e.g., 0.1 for 90% CI). + /// + public double Alpha { get; set; } + + /// + /// Results for each statistic (indexed by statistic index from StatisticFunction output). + /// + public BootstrapStatisticResult[] StatisticResults { get; set; } = Array.Empty(); + + /// + /// Results for each model parameter (indexed by parameter index from ParameterSet.Values). Uses percentile CIs. + /// + public BootstrapStatisticResult[] ParameterResults { get; set; } = Array.Empty(); + + /// + /// The total number of replicates that failed after all retries. + /// + public int FailedReplicates { get; set; } + } +} diff --git a/Numerics/Sampling/MCMC/ARWMH.cs b/Numerics/Sampling/MCMC/ARWMH.cs index 59bd92ab..65c60dac 100644 --- a/Numerics/Sampling/MCMC/ARWMH.cs +++ b/Numerics/Sampling/MCMC/ARWMH.cs @@ -75,7 +75,7 @@ public ARWMH(List priorDistributions, LogLikelihood log } - private Matrix sigmaIdentity; + private Matrix sigmaIdentity = null!; private RunningCovarianceMatrix[] sigma = null!; private MultivariateNormal[] mvn = null!; @@ -127,7 +127,7 @@ protected override void InitializeCustomSettings() mvn[i] = new MultivariateNormal(NumberOfParameters); sigma[i] = new RunningCovarianceMatrix(NumberOfParameters); - if (Initialize == InitializationType.MAP && _mapSuccessful) + if (Initialize == InitializationType.MAP && _mapSuccessful && _MVN != null) { // Hot start the covariance matrix for (int j = 0; j < NumberOfParameters; j++) diff --git a/Numerics/Sampling/MCMC/Base/MCMCSampler.cs b/Numerics/Sampling/MCMC/Base/MCMCSampler.cs index 0c92873a..d0b63f8c 100644 --- a/Numerics/Sampling/MCMC/Base/MCMCSampler.cs +++ b/Numerics/Sampling/MCMC/Base/MCMCSampler.cs @@ -82,11 +82,29 @@ public MCMCSampler(List priorDistributions, LogLikeliho #region Inputs + /// + /// The pseudo random number generator (PRNG) seed. + /// protected int _prngSeed = 12345; + /// + /// The number of initial iterations before adaptation begins. + /// protected int _initialIterations = 10; + /// + /// The number of warmup (burn-in) iterations to discard. + /// protected int _warmupIterations = 1750; + /// + /// The number of sampling iterations after warmup. + /// protected int _iterations = 3500; + /// + /// The number of parallel Markov chains to run. + /// protected int _numberOfChains = 4; + /// + /// The thinning interval for reducing autocorrelation. + /// protected int _thinningInterval = 20; /// @@ -184,7 +202,7 @@ public int ThinningInterval protected Random[] _chainPRNGs = null!; /// - /// The current states of each chain. + /// The current states of each chain. /// protected ParameterSet[] _chainStates = null!; @@ -248,15 +266,20 @@ public enum InitializationType /// protected bool _mapSuccessful = false; + /// + /// Indicates whether MAP initialization was attempted but failed, resulting in a fallback to random initialization. + /// + public bool MAPInitializationFailed { get; protected set; } = false; + /// /// The Multivariate Normal proposal distribution set from the MAP estimate. /// - protected MultivariateNormal _MVN = null!; + protected MultivariateNormal? _MVN; /// /// Event is raised when the simulation progress changes. /// - public event ProgressChangedEventHandler ProgressChanged = null!; + public event ProgressChangedEventHandler? ProgressChanged; /// /// Event is raised when the simulation progress changes. @@ -273,7 +296,7 @@ public enum InitializationType /// /// Cancellation token source. /// - public CancellationTokenSource CancellationTokenSource { get; set; } = null!; + public CancellationTokenSource? CancellationTokenSource { get; set; } #endregion @@ -308,7 +331,7 @@ public double[] AcceptanceRates { var ar = new double[NumberOfChains]; for (int i = 0; i < NumberOfChains; i++) - ar[i] = (double)AcceptCount[i] / (double)SampleCount[i]; + ar[i] = SampleCount[i] > 0 ? (double)AcceptCount[i] / (double)SampleCount[i] : 0d; return ar; } } @@ -324,13 +347,13 @@ public double[] AcceptanceRates public int OutputLength { get; set; } = 10000; /// - /// Output posterior parameter sets. These are recorded after the iterations have been completed. + /// Output posterior parameter sets. These are recorded after the iterations have been completed. /// public List[] Output { get; protected set; } = null!; /// - /// The output parameter set that produced the maximum likelihood. - /// This is referred to as the maximum a posteriori (MAP). + /// The output parameter set that produced the maximum likelihood. + /// This is referred to as the maximum a posteriori (MAP). /// public ParameterSet MAP { get; protected set; } @@ -403,6 +426,7 @@ protected virtual ParameterSet[] InitializeChains() // Get MAP MAP = DE.BestParameterSet.Clone(); // Get Fisher Information Matrix + if (DE.Hessian == null) throw new InvalidOperationException("Hessian matrix is not available."); var fisher = DE.Hessian * -1d; // Invert it to get the covariance matrix, and scale to give wider coverage var covar = fisher.Inverse() * 2; @@ -425,10 +449,11 @@ protected virtual ParameterSet[] InitializeChains() return initials; } - catch (Exception) + catch (Exception) { // If this fails go to naive initialization below Initialize = InitializationType.Randomize; + MAPInitializationFailed = true; } } } @@ -571,7 +596,7 @@ public virtual void Sample() // Update progress progress += 1; - if (progress % (int)(totalIterations * ProgressChangedRate) == 0) + if (progress % Math.Max(1, (int)(totalIterations * ProgressChangedRate)) == 0) { ReportProgress((double)progress / totalIterations); } @@ -607,6 +632,7 @@ public void ReportProgress(double percentComplete) public void Reset() { _simulations = 0; + MAPInitializationFailed = false; // Clear old memory and re-instantiate the result storage _masterPRNG = new Random(PRNGSeed); _chainPRNGs = new Random[NumberOfChains]; @@ -622,7 +648,7 @@ public void Reset() AcceptCount = new int[NumberOfChains]; SampleCount = new int[NumberOfChains]; MeanLogLikelihood = new List(); - MAP = new ParameterSet([], double.MinValue); + MAP = new ParameterSet([], double.MinValue); } #endregion diff --git a/Numerics/Sampling/MCMC/DEMCz.cs b/Numerics/Sampling/MCMC/DEMCz.cs index 93a18211..17353d02 100644 --- a/Numerics/Sampling/MCMC/DEMCz.cs +++ b/Numerics/Sampling/MCMC/DEMCz.cs @@ -131,7 +131,8 @@ protected override ParameterSet ChainIteration(int index, ParameterSet state) // Sample uniformly at random without replacement two numbers R1 and R2 // from the numbers 1, 2, ..., M. int r1, r2, M = PopulationMatrix.Count; - r1 = _chainPRNGs[index].Next(0, M); + if (M < 2) throw new InvalidOperationException("PopulationMatrix must contain at least 2 elements."); + r1 = _chainPRNGs[index].Next(0, M); do r2 = _chainPRNGs[index].Next(0, M); while (r2 == r1); // Calculate the proposal vector diff --git a/Numerics/Sampling/MCMC/DEMCzs.cs b/Numerics/Sampling/MCMC/DEMCzs.cs index 2fd6fe3a..fb0d531e 100644 --- a/Numerics/Sampling/MCMC/DEMCzs.cs +++ b/Numerics/Sampling/MCMC/DEMCzs.cs @@ -202,7 +202,6 @@ private ParameterSet SnookerUpdate(int index, ParameterSet state) // Do snooker update // Get Jump -- uniform random number between 1.2 and 2.2 double G = _g.InverseCDF(_chainPRNGs[index].NextDouble()); - //double G = 1.7; // Select another chain, which is in state z int c = index; diff --git a/Numerics/Sampling/MCMC/HMC.cs b/Numerics/Sampling/MCMC/HMC.cs index 36ee7181..a0dcbfa1 100644 --- a/Numerics/Sampling/MCMC/HMC.cs +++ b/Numerics/Sampling/MCMC/HMC.cs @@ -104,10 +104,19 @@ public HMC(List priorDistributions, LogLikelihood logLi StepSize = stepSize; Steps = steps; - // Set the gradient function + // Cache prior distribution bounds for the gradient function + _lowerBounds = new double[NumberOfParameters]; + _upperBounds = new double[NumberOfParameters]; + for (int i = 0; i < NumberOfParameters; i++) + { + _lowerBounds[i] = priorDistributions[i].Minimum; + _upperBounds[i] = priorDistributions[i].Maximum; + } + + // Set the gradient function with prior bounds so finite-difference probes stay in valid region if (gradientFunction == null) { - GradientFunction = (x) => new Vector(NumericalDerivative.Gradient((y) => LogLikelihoodFunction(y), x.ToArray())); + GradientFunction = (x) => new Vector(NumericalDerivative.Gradient((y) => SafeLogLikelihood(y), x.ToArray(), _lowerBounds, _upperBounds)); } else { @@ -121,6 +130,8 @@ public HMC(List priorDistributions, LogLikelihood logLi private Vector _inverseMass; private double _stepSize = 0.1; private int _steps = 50; + private double[] _lowerBounds; + private double[] _upperBounds; /// /// The mass vector for the momentum distribution. @@ -189,64 +200,98 @@ protected override ParameterSet ChainIteration(int index, ParameterSet state) // Update the sample count SampleCount[index] += 1; - // Jigger the step size and number of steps - var _stepSize = _stepSizeU.InverseCDF(_chainPRNGs[index].NextDouble()); - var _steps = (int)Math.Ceiling(_stepsU.InverseCDF(_chainPRNGs[index].NextDouble())); - - // Step 1. Sample phi from a N~(0,M) - var phi = new Vector(NumberOfParameters); - for (int i = 0; i < NumberOfParameters; i++) - phi[i] = Math.Sqrt(Mass[i]) * Normal.StandardZ(_chainPRNGs[index].NextDouble()); + try + { + // Jigger the step size and number of steps + var _stepSize = _stepSizeU.InverseCDF(_chainPRNGs[index].NextDouble()); + var _steps = (int)Math.Ceiling(_stepsU.InverseCDF(_chainPRNGs[index].NextDouble())); - // Get kinetic energy of the current state - var logKi = -0.5 * QuadraticForm(phi, _inverseMass); + // Step 1. Sample phi from a N~(0,M) + var phi = new Vector(NumberOfParameters); + for (int i = 0; i < NumberOfParameters; i++) + phi[i] = Math.Sqrt(Mass[i]) * Normal.StandardZ(_chainPRNGs[index].NextDouble()); - // Step 2. Perform leapfrog steps to get proposal vector - var xp = new Vector(state.Values); - phi += GradientFunction(xp.Array) * _stepSize * 0.5; - for (int i = 0; i < _steps; i++) - { - xp += _inverseMass * phi * _stepSize; + // Get kinetic energy of the current state + var logKi = -0.5 * QuadraticForm(phi, _inverseMass); - // Ensure the parameters are feasible (within the constraints) - for (int j = 0; j < NumberOfParameters; j++) + // Step 2. Perform leapfrog steps to get proposal vector + var xp = new Vector(state.Values); + phi += GradientFunction(xp.Array) * _stepSize * 0.5; + for (int i = 0; i < _steps; i++) { - if (xp[j] < PriorDistributions[j].Minimum) - xp[j] = PriorDistributions[j].Minimum + Tools.DoubleMachineEpsilon; - if (xp[j] > PriorDistributions[j].Maximum) - xp[j] = PriorDistributions[j].Maximum - Tools.DoubleMachineEpsilon; - } + xp += _inverseMass * phi * _stepSize; - phi += GradientFunction(xp.Array) * _stepSize * (i == _steps-1 ? 0.5: 1.0); - } - phi *= -1d; + // Ensure the parameters are feasible (within the constraints) + for (int j = 0; j < NumberOfParameters; j++) + { + if (xp[j] < PriorDistributions[j].Minimum) + { + xp[j] = PriorDistributions[j].Minimum + Tools.DoubleMachineEpsilon; + phi[j] = -phi[j]; + } + if (xp[j] > PriorDistributions[j].Maximum) + { + xp[j] = PriorDistributions[j].Maximum - Tools.DoubleMachineEpsilon; + phi[j] = -phi[j]; + } + } - // Get kinetic energy of the proposal state - var logKp = -0.5 * QuadraticForm(phi, _inverseMass); + phi += GradientFunction(xp.Array) * _stepSize * (i == _steps - 1 ? 0.5 : 1.0); + } + phi *= -1d; - // Evaluate fitness - var logLHp = LogLikelihoodFunction(xp.Array); - var logLHi = state.Fitness; + // Get kinetic energy of the proposal state + var logKp = -0.5 * QuadraticForm(phi, _inverseMass); - // Calculate the Metropolis ratio - var logRatio = logLHp - logLHi + logKp - logKi; + // Evaluate fitness + var logLHp = SafeLogLikelihood(xp.Array); + var logLHi = state.Fitness; - // Accept the proposal with probability min(1,r) - // otherwise leave xi unchanged - var logU = Math.Log(_chainPRNGs[index].NextDouble()); - if (logU <= logRatio) - { - // The proposal is accepted - AcceptCount[index] += 1; - return new ParameterSet(xp.Array, logLHp); + // Calculate the Metropolis ratio + var logRatio = logLHp - logLHi + logKp - logKi; + + // Accept the proposal with probability min(1,r) + // otherwise leave xi unchanged + var logU = Math.Log(_chainPRNGs[index].NextDouble()); + if (logU <= logRatio) + { + // The proposal is accepted + AcceptCount[index] += 1; + return new ParameterSet(xp.Array, logLHp); + } + else + { + return state; + } } - else + catch (ArithmeticException) { + // Non-finite gradient encountered during leapfrog integration. + // This occurs when parameters drift into regions where the log-likelihood + // returns -Infinity. Reject the proposal and return the current state, + // consistent with Metropolis rejection behavior. return state; } } + /// + /// Evaluates the log-likelihood, returning negative infinity if the parameters are out of range. + /// This prevents ArgumentOutOfRangeException from propagating during leapfrog integration + /// when the sampler explores parameter values that violate distribution constraints. + /// + private double SafeLogLikelihood(double[] parameters) + { + try + { + return LogLikelihoodFunction(parameters); + } + catch (ArgumentOutOfRangeException) + { + return double.NegativeInfinity; + } + } + /// /// Computes the quadratic form φᵀ M⁻¹ φ, which is used to calculate the kinetic energy /// in Hamiltonian Monte Carlo (HMC) sampling. This method avoids allocating intermediate arrays. diff --git a/Numerics/Sampling/MCMC/NUTS.cs b/Numerics/Sampling/MCMC/NUTS.cs new file mode 100644 index 00000000..bf83ed8a --- /dev/null +++ b/Numerics/Sampling/MCMC/NUTS.cs @@ -0,0 +1,516 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using Numerics.Distributions; +using Numerics.Mathematics; +using Numerics.Mathematics.LinearAlgebra; +using Numerics.Mathematics.Optimization; +using System; +using System.Collections.Generic; + +namespace Numerics.Sampling.MCMC +{ + + /// + /// The No-U-Turn Sampler (NUTS), an adaptive extension of Hamiltonian Monte Carlo that + /// automatically tunes the trajectory length. + /// + /// + /// + /// Authors: + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// Description: + /// + /// + /// NUTS eliminates the need to manually tune the number of leapfrog steps required by standard HMC. + /// It builds a balanced binary tree of leapfrog states by repeatedly doubling the trajectory in a + /// randomly chosen direction. The trajectory stops when a U-turn is detected (the trajectory starts + /// heading back towards the starting point) or when a maximum tree depth is reached. + /// A candidate state is selected via multinomial sampling weighted by the exponential of the + /// negative Hamiltonian. + /// + /// + /// During the warmup phase, the leapfrog step size is automatically adapted using dual averaging + /// to achieve a target Metropolis acceptance probability of approximately 80%. + /// + /// + /// Key applications include: + /// + /// Bayesian flood frequency analysis in RMC-BestFit where manual HMC tuning is impractical. + /// Bayesian parameter estimation for complex hydrologic models in RFA. + /// Posterior inference for mixture models and hierarchical models in TotalRisk. + /// + /// + /// + /// References: + /// + /// + /// + /// + /// Hoffman, M.D. and Gelman, A. (2014). "The No-U-Turn Sampler: Adaptively Setting Path Lengths + /// in Hamiltonian Monte Carlo." Journal of Machine Learning Research, 15, 1593-1623. + /// + /// + /// Betancourt, M. (2017). "A Conceptual Introduction to Hamiltonian Monte Carlo." arXiv:1701.02434. + /// + /// + /// + /// + /// + /// + /// + [Serializable] + public class NUTS : MCMCSampler + { + + /// + /// Constructs a new NUTS sampler. + /// + /// The list of prior distributions for the model parameters. + /// The log-likelihood function to evaluate. + /// Optional. The mass vector for the momentum distribution. Default = Identity. + /// Optional. The initial leapfrog step size. Will be adapted during warmup. Default = 0.1. + /// Optional. The maximum binary tree depth. Default = 10. + /// Optional. The function for evaluating the gradient of the log-likelihood. Numerical finite difference will be used by default. + public NUTS(List priorDistributions, LogLikelihood logLikelihoodFunction, Vector? mass = null, double stepSize = 0.1, int maxTreeDepth = 10, HMC.Gradient? gradientFunction = null) : base(priorDistributions, logLikelihoodFunction) + { + InitialIterations = 100 * NumberOfParameters; + + // Set the mass vector + if (mass == null) + { + Mass = new Vector(NumberOfParameters, 1d); + } + else + { + Mass = mass; + } + + // Set the inverse mass vector + _inverseMass = new Vector(NumberOfParameters); + for (int i = 0; i < NumberOfParameters; i++) + _inverseMass[i] = 1d / Mass[i]; + + // Set defaults + _initialStepSize = stepSize; + MaxTreeDepth = maxTreeDepth; + + // Cache prior distribution bounds for the gradient function + _lowerBounds = new double[NumberOfParameters]; + _upperBounds = new double[NumberOfParameters]; + for (int i = 0; i < NumberOfParameters; i++) + { + _lowerBounds[i] = priorDistributions[i].Minimum; + _upperBounds[i] = priorDistributions[i].Maximum; + } + + // Set the gradient function with prior bounds so finite-difference probes stay in valid region + if (gradientFunction == null) + { + GradientFunction = (x) => new Vector(NumericalDerivative.Gradient((y) => SafeLogLikelihood(y), x.ToArray(), _lowerBounds, _upperBounds)); + } + else + { + GradientFunction = gradientFunction; + } + } + + private Vector _inverseMass; + private double _initialStepSize; + private double[] _lowerBounds; + private double[] _upperBounds; + + // Per-chain dual averaging state + private double[] _chainStepSizes = null!; + private double[] _chainLogEpsBar = null!; + private double[] _chainHBar = null!; + private double[] _chainMu = null!; + private int[] _chainAdaptStep = null!; + + // Dual averaging hyperparameters (Hoffman & Gelman 2014, Section 3.2) + private const double DELTA_TARGET = 0.80; + private const double GAMMA = 0.05; + private const double T0 = 10.0; + private const double KAPPA = 0.75; + + // Divergence threshold: if H - H0 exceeds this, the trajectory is considered divergent + private const double MAX_DELTA_H = 1000.0; + + /// + /// The mass vector for the momentum distribution. + /// + public Vector Mass { get; } + + /// + /// The maximum binary tree depth. Default = 10. This caps the trajectory length at 2^MaxTreeDepth leapfrog steps. + /// + public int MaxTreeDepth { get; set; } + + /// + /// The function for evaluating the gradient of the log-likelihood. + /// + public HMC.Gradient GradientFunction { get; } + + /// + /// The target Metropolis acceptance probability for dual averaging adaptation. Default = 0.80. + /// + public double TargetAcceptanceRate => DELTA_TARGET; + + /// + protected override void ValidateCustomSettings() + { + if (Mass.Length != NumberOfParameters) throw new ArgumentException(nameof(Mass), "The mass vector must be the same length as the number of parameters."); + if (_initialStepSize <= 0) throw new ArgumentException("stepSize", "The leapfrog step size must be positive."); + if (MaxTreeDepth < 1) throw new ArgumentException(nameof(MaxTreeDepth), "The maximum tree depth must be at least 1."); + } + + /// + protected override void InitializeCustomSettings() + { + _chainStepSizes = new double[NumberOfChains]; + _chainLogEpsBar = new double[NumberOfChains]; + _chainHBar = new double[NumberOfChains]; + _chainMu = new double[NumberOfChains]; + _chainAdaptStep = new int[NumberOfChains]; + + for (int i = 0; i < NumberOfChains; i++) + { + _chainStepSizes[i] = _initialStepSize; + _chainLogEpsBar[i] = Math.Log(_initialStepSize); + _chainHBar[i] = 0.0; + _chainMu[i] = Math.Log(10.0 * _initialStepSize); + _chainAdaptStep[i] = 0; + } + + } + + /// + protected override ParameterSet ChainIteration(int index, ParameterSet state) + { + // Update the sample count + SampleCount[index] += 1; + + double eps = _chainStepSizes[index]; + + // Step 1: Sample momentum from N(0, M) + var phi = new Vector(NumberOfParameters); + for (int i = 0; i < NumberOfParameters; i++) + phi[i] = Math.Sqrt(Mass[i]) * Normal.StandardZ(_chainPRNGs[index].NextDouble()); + + // Compute initial Hamiltonian: H = -log p(theta) + 0.5 * phi^T M^{-1} phi + double H0 = -state.Fitness + 0.5 * HMC.QuadraticForm(phi, _inverseMass); + + // Step 2: Initialize tree + var theta = new Vector(state.Values); + var thetaMinus = theta.Clone(); + var thetaPlus = theta.Clone(); + var rMinus = phi.Clone(); + var rPlus = phi.Clone(); + + var candidate = theta.Clone(); + double candidateLogLH = state.Fitness; + double logSumWeight = -H0; + + int depth = 0; + double sumAlpha = 0; + int numAlpha = 0; + + // Step 3: Build tree by doubling until U-turn or max depth + while (depth < MaxTreeDepth) + { + // Choose a random direction + int v = _chainPRNGs[index].NextDouble() < 0.5 ? -1 : 1; + + TreeState subtree; + if (v == -1) + { + subtree = BuildTree(thetaMinus, rMinus, -eps, depth, H0, index); + thetaMinus = subtree.ThetaMinus; + rMinus = subtree.MomentumMinus; + } + else + { + subtree = BuildTree(thetaPlus, rPlus, eps, depth, H0, index); + thetaPlus = subtree.ThetaPlus; + rPlus = subtree.MomentumPlus; + } + + // If the subtree is valid, consider accepting its candidate + if (subtree.Valid) + { + double logSumWeightNew = LogSumExp(logSumWeight, subtree.LogSumWeight); + double acceptProb = Math.Exp(subtree.LogSumWeight - logSumWeightNew); + if (_chainPRNGs[index].NextDouble() < acceptProb) + { + candidate = subtree.ThetaPrime; + candidateLogLH = subtree.LogLikelihoodPrime; + } + logSumWeight = logSumWeightNew; + } + + // Accumulate adaptation statistics + sumAlpha += subtree.SumAlpha; + numAlpha += subtree.NumAlpha; + + // Check stopping criterion: divergence or U-turn at the top level + if (!subtree.Valid) + break; + + var dTheta = thetaPlus - thetaMinus; + if (Vector.DotProduct(dTheta, rMinus) < 0 || Vector.DotProduct(dTheta, rPlus) < 0) + break; + + depth++; + } + + // Step 4: Dual averaging step size adaptation during warmup + int warmupSteps = WarmupIterations * ThinningInterval; + if (SampleCount[index] <= warmupSteps) + { + double avgAlpha = numAlpha > 0 ? sumAlpha / numAlpha : DELTA_TARGET; + DualAveragingUpdate(index, avgAlpha); + } + else if (SampleCount[index] == warmupSteps + 1) + { + // After warmup, fix step size to the smoothed value + _chainStepSizes[index] = Math.Exp(_chainLogEpsBar[index]); + } + + // NUTS always accepts + AcceptCount[index] += 1; + return new ParameterSet(candidate.Array, candidateLogLH); + } + + /// + /// Recursively builds a balanced binary tree of leapfrog states. + /// + /// Starting position. + /// Starting momentum. + /// Signed step size (negative = backward direction). + /// Current tree depth (0 = single leapfrog step). + /// Initial Hamiltonian for the trajectory. + /// The chain index for RNG access. + /// The tree state containing endpoints, candidate, and diagnostics. + private TreeState BuildTree(Vector theta, Vector momentum, double epsilon, int depth, double H0, int chainIndex) + { + if (depth == 0) + { + // Base case: take one leapfrog step + var (thetaPrime, momentumPrime) = Leapfrog(theta, momentum, epsilon); + double logLH = SafeLogLikelihood(thetaPrime.Array); + double H = -logLH + 0.5 * HMC.QuadraticForm(momentumPrime, _inverseMass); + double logWeight = -H; + bool divergent = (H - H0) > MAX_DELTA_H; + double alpha = Math.Min(1.0, Math.Exp(H0 - H)); + if (double.IsNaN(alpha)) alpha = 0; + + return new TreeState + { + ThetaMinus = thetaPrime, + MomentumMinus = momentumPrime, + ThetaPlus = thetaPrime, + MomentumPlus = momentumPrime, + ThetaPrime = thetaPrime, + LogSumWeight = logWeight, + LogLikelihoodPrime = logLH, + LeafCount = 1, + Valid = !divergent, + SumAlpha = alpha, + NumAlpha = 1 + }; + } + + // Recursive case: build first half-tree + var tree = BuildTree(theta, momentum, epsilon, depth - 1, H0, chainIndex); + + if (tree.Valid) + { + // Build second half-tree + TreeState tree2; + if (epsilon > 0) + { + tree2 = BuildTree(tree.ThetaPlus, tree.MomentumPlus, epsilon, depth - 1, H0, chainIndex); + tree.ThetaPlus = tree2.ThetaPlus; + tree.MomentumPlus = tree2.MomentumPlus; + } + else + { + tree2 = BuildTree(tree.ThetaMinus, tree.MomentumMinus, epsilon, depth - 1, H0, chainIndex); + tree.ThetaMinus = tree2.ThetaMinus; + tree.MomentumMinus = tree2.MomentumMinus; + } + + // Multinomial sampling: accept candidate from tree2 with appropriate probability + double logSumWeightNew = LogSumExp(tree.LogSumWeight, tree2.LogSumWeight); + double acceptTree2Prob = Math.Exp(tree2.LogSumWeight - logSumWeightNew); + if (_chainPRNGs[chainIndex].NextDouble() < acceptTree2Prob) + { + tree.ThetaPrime = tree2.ThetaPrime; + tree.LogLikelihoodPrime = tree2.LogLikelihoodPrime; + } + + tree.LogSumWeight = logSumWeightNew; + tree.LeafCount += tree2.LeafCount; + tree.SumAlpha += tree2.SumAlpha; + tree.NumAlpha += tree2.NumAlpha; + + // Check U-turn criterion on the combined tree + var dTheta = tree.ThetaPlus - tree.ThetaMinus; + bool uturn = Vector.DotProduct(dTheta, tree.MomentumMinus) < 0 || + Vector.DotProduct(dTheta, tree.MomentumPlus) < 0; + tree.Valid = tree2.Valid && !uturn; + } + + return tree; + } + + /// + /// Performs a single leapfrog integration step with boundary enforcement. + /// + /// Current position. + /// Current momentum. + /// The step size (signed: positive = forward, negative = backward). + /// The updated (position, momentum) after one leapfrog step. + private (Vector theta, Vector momentum) Leapfrog(Vector theta, Vector momentum, double epsilon) + { + // Half-step momentum update + var grad = GradientFunction(theta.Array); + var r = momentum + grad * (epsilon * 0.5); + + // Full-step position update + var q = theta + _inverseMass * r * epsilon; + + // Enforce parameter bounds + for (int j = 0; j < NumberOfParameters; j++) + { + if (q[j] < PriorDistributions[j].Minimum) + q[j] = PriorDistributions[j].Minimum + Tools.DoubleMachineEpsilon; + if (q[j] > PriorDistributions[j].Maximum) + q[j] = PriorDistributions[j].Maximum - Tools.DoubleMachineEpsilon; + } + + // Half-step momentum update + grad = GradientFunction(q.Array); + r = r + grad * (epsilon * 0.5); + + return (q, r); + } + + /// + /// Updates the step size using the dual averaging scheme from Hoffman and Gelman (2014), Algorithm 5. + /// + /// The chain index. + /// The average Metropolis acceptance probability from the current tree. + private void DualAveragingUpdate(int chainIndex, double avgAcceptProb) + { + _chainAdaptStep[chainIndex]++; + int m = _chainAdaptStep[chainIndex]; + + // Update running average of the acceptance statistic + _chainHBar[chainIndex] = (1.0 - 1.0 / (m + T0)) * _chainHBar[chainIndex] + + (DELTA_TARGET - avgAcceptProb) / (m + T0); + + // Compute new log step size + double logEps = _chainMu[chainIndex] - Math.Sqrt(m) / GAMMA * _chainHBar[chainIndex]; + + // Update smoothed log step size (exponential moving average) + double mPow = Math.Pow(m, -KAPPA); + _chainLogEpsBar[chainIndex] = mPow * logEps + (1.0 - mPow) * _chainLogEpsBar[chainIndex]; + + // Set current step size (during adaptation, use the un-smoothed value) + _chainStepSizes[chainIndex] = Math.Exp(logEps); + + // Clamp step size to prevent extreme values + if (_chainStepSizes[chainIndex] < 1e-10) + _chainStepSizes[chainIndex] = 1e-10; + if (_chainStepSizes[chainIndex] > 1e5) + _chainStepSizes[chainIndex] = 1e5; + } + + /// + /// Computes log(exp(a) + exp(b)) in a numerically stable way. + /// + private static double LogSumExp(double a, double b) + { + double max = Math.Max(a, b); + if (double.IsNegativeInfinity(max)) return double.NegativeInfinity; + return max + Math.Log(Math.Exp(a - max) + Math.Exp(b - max)); + } + + /// + /// Evaluates the log-likelihood, returning negative infinity if the parameters are out of range. + /// This prevents ArgumentOutOfRangeException from propagating during leapfrog integration + /// when the sampler explores parameter values that violate distribution constraints. + /// + private double SafeLogLikelihood(double[] parameters) + { + try + { + return LogLikelihoodFunction(parameters); + } + catch (ArgumentOutOfRangeException) + { + return double.NegativeInfinity; + } + } + + /// + /// Internal state of a binary tree node used during the NUTS tree-building recursion. + /// + private struct TreeState + { + /// Leftmost position in the subtree. + public Vector ThetaMinus; + /// Leftmost momentum in the subtree. + public Vector MomentumMinus; + /// Rightmost position in the subtree. + public Vector ThetaPlus; + /// Rightmost momentum in the subtree. + public Vector MomentumPlus; + /// Candidate position selected by multinomial sampling. + public Vector ThetaPrime; + /// Log of the sum of weights for multinomial sampling. + public double LogSumWeight; + /// Log-likelihood of the candidate position. + public double LogLikelihoodPrime; + /// Number of leaf nodes in the subtree. + public int LeafCount; + /// Whether the subtree is valid (no divergence, no U-turn). + public bool Valid; + /// Sum of per-leaf Metropolis acceptance probabilities (for dual averaging). + public double SumAlpha; + /// Number of leaves contributing to SumAlpha. + public int NumAlpha; + } + + } +} diff --git a/Numerics/Sampling/MCMC/RWMH.cs b/Numerics/Sampling/MCMC/RWMH.cs index d3801c8c..35f04dac 100644 --- a/Numerics/Sampling/MCMC/RWMH.cs +++ b/Numerics/Sampling/MCMC/RWMH.cs @@ -94,9 +94,9 @@ protected override void InitializeCustomSettings() mvn[i] = new MultivariateNormal(NumberOfParameters); } // Set up proposal matrix - if (Initialize == InitializationType.MAP && _mapSuccessful) + if (Initialize == InitializationType.MAP && _mapSuccessful && _MVN != null) { - ProposalSigma = new Matrix(base._MVN.Covariance); + ProposalSigma = new Matrix(_MVN.Covariance); } } diff --git a/Numerics/Sampling/MCMC/SNIS.cs b/Numerics/Sampling/MCMC/SNIS.cs index 5a0561a5..48503d71 100644 --- a/Numerics/Sampling/MCMC/SNIS.cs +++ b/Numerics/Sampling/MCMC/SNIS.cs @@ -80,7 +80,7 @@ public SNIS(List priorDistributions, LogLikelihood logL /// protected override void InitializeCustomSettings() { - if (mvn == null && useImportanceSampling == false && Initialize == InitializationType.MAP && _mapSuccessful) + if (mvn == null && useImportanceSampling == false && Initialize == InitializationType.MAP && _mapSuccessful && _MVN != null) { mvn = (MultivariateNormal)_MVN.Clone(); var covar = new Matrix(mvn.Covariance); @@ -149,7 +149,7 @@ public override void Sample() var parameters = new double[NumberOfParameters]; double logLH = 0; double weight = 0; - if (useImportanceSampling == true) + if (useImportanceSampling == true && mvn != null) { parameters = mvn.InverseCDF(rnds.GetRow(idx)); logLH = LogLikelihoodFunction(parameters); @@ -176,7 +176,7 @@ public override void Sample() double sum = 0; for (int i = 0; i < Iterations; i++) { - if (!double.IsNaN(MarkovChains[0][i].Weight) || MarkovChains[0][i].Weight != double.MinValue) + if (!double.IsNaN(MarkovChains[0][i].Weight) && MarkovChains[0][i].Weight != double.MinValue) { sum += Math.Exp(MarkovChains[0][i].Weight - max); } @@ -186,7 +186,7 @@ public override void Sample() // Compute the posterior weights Parallel.For(0, Iterations, (idx) => { - double w = !double.IsNaN(MarkovChains[0][idx].Weight) || MarkovChains[0][idx].Weight != double.MinValue ? Math.Exp(MarkovChains[0][idx].Weight - normalization) : 0d; + double w = !double.IsNaN(MarkovChains[0][idx].Weight) && MarkovChains[0][idx].Weight != double.MinValue ? Math.Exp(MarkovChains[0][idx].Weight - normalization) : 0d; MarkovChains[0][idx] = new ParameterSet(MarkovChains[0][idx].Values, MarkovChains[0][idx].Fitness, w); }); diff --git a/Numerics/Sampling/MCMC/Support/MCMCDiagnostics.cs b/Numerics/Sampling/MCMC/Support/MCMCDiagnostics.cs index d3e451d7..c7f651a3 100644 --- a/Numerics/Sampling/MCMC/Support/MCMCDiagnostics.cs +++ b/Numerics/Sampling/MCMC/Support/MCMCDiagnostics.cs @@ -59,6 +59,7 @@ public static double EffectiveSampleSize(IList series) //https://www.rdocumentation.org/packages/LaplacesDemon/versions/16.1.4/topics/ESS int N = series.Count; var acf = Fourier.Autocorrelation(series, (int)Math.Ceiling((double)N / 2)); + if (acf == null) return N; double rho = 0; for (int i = 1; i < acf.GetLength(0); i++) { @@ -105,6 +106,7 @@ public static double[] EffectiveSampleSize(IList> markovChain // Get ACF for this chain var acf = Fourier.Autocorrelation(values, (int)Math.Ceiling((double)N / 2)); + if (acf == null) continue; // Update the average ACF across all chains for (int j = 0; j < acf.GetLength(0); j++) { @@ -182,13 +184,14 @@ public static double[] GelmanRubin(IList> markovChains, int w chainMeans[i] += markovChains[i][j].Values[p]; } // Get within-chain mean - chainMeans[i] /= N; + chainMeans[i] /= (N - startIndex); overallMean += chainMeans[i]; } // Get between-chain mean overallMean /= M; // Step 2. Compute between- and within-chain variance + int n = N - startIndex; double B = 0, W = 0; for (int i = 0; i < M; i++) { @@ -198,16 +201,16 @@ public static double[] GelmanRubin(IList> markovChains, int w sum += Tools.Sqr(markovChains[i][j].Values[p] - chainMeans[i]); } // within-chain variance - W += sum / (N - 1); + W += sum / (n - 1); // between-chain variance B += Tools.Sqr(chainMeans[i] - overallMean); } // Set between- and within-chain variance W /= M; - B *= N / (M - 1); + B *= n / (double)(M - 1); // Step 3. Compute the pooled variance - double V = ((N - 1d) / N) * W + (1d / N) * B; + double V = ((n - 1d) / n) * W + (1d / n) * B; // Step 4. Compute R-hat Rhat[p] = Math.Sqrt(V / W); diff --git a/Numerics/Sampling/MCMC/Support/MCMCResults.cs b/Numerics/Sampling/MCMC/Support/MCMCResults.cs index 7ba6bb2a..436f412d 100644 --- a/Numerics/Sampling/MCMC/Support/MCMCResults.cs +++ b/Numerics/Sampling/MCMC/Support/MCMCResults.cs @@ -29,9 +29,9 @@ */ using Numerics.Mathematics.Optimization; +using Numerics.Utilities; using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; @@ -94,19 +94,19 @@ public MCMCResults(ParameterSet map, IList parameterSets, double a /// The list of sampled Markov Chains. /// [JsonInclude] - public List[] MarkovChains { get; private set; } = null!; + public List[]? MarkovChains { get; private set; } /// /// Output posterior parameter sets. /// [JsonInclude] - public List Output { get; private set; } = null!; + public List Output { get; private set; } = new List(); /// /// The average log-likelihood across each chain for each iteration. /// [JsonInclude] - public List MeanLogLikelihood { get; private set; } = null!; + public List? MeanLogLikelihood { get; private set; } /// /// The acceptance rate for each chain. @@ -125,13 +125,13 @@ public MCMCResults(ParameterSet map, IList parameterSets, double a /// This is referred to as the maximum a posteriori (MAP). /// [JsonInclude] - public ParameterSet MAP { get; private set; } + public ParameterSet MAP { get; private set; } = new ParameterSet(); /// /// The mean of the posterior distribution of each parameter. /// [JsonInclude] - public ParameterSet PosteriorMean { get; private set; } + public ParameterSet PosteriorMean { get; private set; } = new ParameterSet(); /// /// Process the parameter results. @@ -194,6 +194,8 @@ public static byte[] ToByteArray(MCMCResults mcmcResults) DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, IncludeFields = true }; + options.Converters.Add(new Double2DArrayConverter()); + options.Converters.Add(new HistogramConverter()); return JsonSerializer.SerializeToUtf8Bytes(mcmcResults, options); } @@ -201,40 +203,16 @@ public static byte[] ToByteArray(MCMCResults mcmcResults) /// Creates MCMC Results from a byte array. /// /// Byte array. - public static MCMCResults FromByteArray(byte[] bytes) + public static MCMCResults? FromByteArray(byte[] bytes) { var options = new JsonSerializerOptions { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, IncludeFields = true }; - try - { - return JsonSerializer.Deserialize(bytes, options) - ?? throw new JsonException("Deserialized MCMCResults was null"); - } - catch - { - ///Previous serialization used Binary Formatter, which won't deserialize cleanly as JSON. - /// If this fails, then it's probably the bf bytes. fall back to legacy. - return FromByteArrayLegacy(bytes); - } - } - - /// - /// Creates MCMC Results from a byte array. - /// - /// Byte array. - private static MCMCResults FromByteArrayLegacy(byte[] bytes) - { - using var ms = new MemoryStream(); - #pragma warning disable SYSLIB0011 // Suppress obsolete BinaryFormatter warning for legacy support - var bf = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter(); - ms.Write(bytes, 0, bytes.Length); - ms.Seek(0L, SeekOrigin.Begin); - var obj = bf.Deserialize(ms); - #pragma warning disable SYSLIB0011 // Suppress obsolete BinaryFormatter warning for legacy support - return (MCMCResults)obj; + options.Converters.Add(new Double2DArrayConverter()); + options.Converters.Add(new HistogramConverter()); + return JsonSerializer.Deserialize(bytes, options); } #endregion diff --git a/Numerics/Sampling/MCMC/Support/ParameterResults.cs b/Numerics/Sampling/MCMC/Support/ParameterResults.cs index 7ac720a0..f7d407eb 100644 --- a/Numerics/Sampling/MCMC/Support/ParameterResults.cs +++ b/Numerics/Sampling/MCMC/Support/ParameterResults.cs @@ -49,6 +49,12 @@ namespace Numerics.Sampling.MCMC public class ParameterResults { + /// + /// Parameterless constructor for JSON deserialization. + /// + [JsonConstructor] + public ParameterResults() { } + /// /// Constructs new parameter results. /// @@ -85,24 +91,24 @@ public ParameterResults(double[] values, double alpha = 0.1, bool sorted = false /// Parameter summary statistics. /// [JsonInclude] - public ParameterStatistics SummaryStatistics { get; private set; } + public ParameterStatistics SummaryStatistics { get; private set; } = null!; /// /// The kernel density results. /// [JsonInclude] - public double[,] KernelDensity { get; private set; } + public double[,] KernelDensity { get; private set; } = new double[0, 0]; /// /// The histogram results. /// [JsonInclude] - public Histogram Histogram { get; private set; } + public Histogram Histogram { get; private set; } = null!; /// /// The autocorrelation function for each parameter. This is averaged across each chain. /// - public double[,] Autocorrelation { get; set; } = null!; + public double[,] Autocorrelation { get; set; } = new double[0, 0]; } } diff --git a/Numerics/Sampling/SobolSequence.cs b/Numerics/Sampling/SobolSequence.cs index 4f6b753e..da8eff6d 100644 --- a/Numerics/Sampling/SobolSequence.cs +++ b/Numerics/Sampling/SobolSequence.cs @@ -136,62 +136,54 @@ private void initialize() reader.ReadLine(); int index = 1; - string? line; - while ((line = reader.ReadLine()) is not null) + string? line = null; + while ((line = reader.ReadLine()) != null) { var st = line.Split(' '); - try - { - int dim; - int.TryParse(st[0], out dim); - - if (dim >= 2 && dim <= Dimension) - { - // we have found the right dimension - int i, s = 0, a = 0; - for (i = 1; i < st.Length; i++) + int dim; + int.TryParse(st[0], out dim); + + if (dim >= 2 && dim <= Dimension) + { + // we have found the right dimension + int i, s = 0, a = 0; + for (i = 1; i < st.Length; i++) + { + if (st[i] != "") { - if (st[i] != "") - { - int.TryParse(st[i], out s); - break; - } + int.TryParse(st[i], out s); + break; } - i++; - for (; i < st.Length; i++) + } + i++; + for (; i < st.Length; i++) + { + if (st[i] != "") { - if (st[i] != "") - { - int.TryParse(st[i], out a); - break; - } + int.TryParse(st[i], out a); + break; } - i++; - int[] m = new int[s + 1]; - for (; i < st.Length; i++) + } + i++; + int[] m = new int[s + 1]; + for (; i < st.Length; i++) + { + if (st[i] != "") { - if (st[i] != "") + for (int j = 1; j <= s; j++) { - for (int j = 1; j <= s; j++) - { - int.TryParse(st[i + j - 1], out m[j]); - } - break; - } + int.TryParse(st[i + j - 1], out m[j]); + } + break; } - initDirectionVector(index++, a, m); - } - - if (dim > Dimension) - { - return; } - + initDirectionVector(index++, a, m); } - catch (Exception) + + if (dim > Dimension) { - throw; + return; } } diff --git a/Numerics/Sampling/StratificationBin.cs b/Numerics/Sampling/StratificationBin.cs index db403ae7..c19788ac 100644 --- a/Numerics/Sampling/StratificationBin.cs +++ b/Numerics/Sampling/StratificationBin.cs @@ -147,9 +147,7 @@ public bool Contains(double x) /// public int CompareTo(StratificationBin? other) { - if (other == null) - throw new ArgumentNullException(nameof(other), "The stratification bin to compare to cannot be null."); - + if (other is null) return 1; if (UpperBound > other.LowerBound && LowerBound < other.UpperBound) throw new ArgumentException("The bins cannot be overlapping.", nameof(other)); diff --git a/Numerics/Sampling/Stratify.cs b/Numerics/Sampling/Stratify.cs index 2519d910..7389c018 100644 --- a/Numerics/Sampling/Stratify.cs +++ b/Numerics/Sampling/Stratify.cs @@ -302,7 +302,7 @@ public static List Probabilities(StratificationOptions option /// The number of dimensions to stratify. /// Seed for random number generator. /// The correlation matrix. If null, independence is assumed. - public static List> MultivariateProbabilities(StratificationOptions options, ImportanceDistribution distributionType = ImportanceDistribution.Uniform, bool isExhaustive = true, int dimension = 1, int seed = -1, double[,] correlation = null!) + public static List> MultivariateProbabilities(StratificationOptions options, ImportanceDistribution distributionType = ImportanceDistribution.Uniform, bool isExhaustive = true, int dimension = 1, int seed = -1, double[,]? correlation = null) { // Validate inputs var output = new List>(); diff --git a/Numerics/Utilities/ExtensionMethods.cs b/Numerics/Utilities/ExtensionMethods.cs index e9f66ad9..8e0ad702 100644 --- a/Numerics/Utilities/ExtensionMethods.cs +++ b/Numerics/Utilities/ExtensionMethods.cs @@ -58,7 +58,7 @@ public static class ExtensionMethods /// The type of the attribute you want to retrieve /// The enum value /// The attribute of type T that exists on the enum value - public static T GetAttributeOfType(this Enum enumValue) where T : Attribute + public static T? GetAttributeOfType(this Enum enumValue) where T : Attribute { var type = enumValue.GetType(); var memInfo = type.GetMember(enumValue.ToString()); @@ -177,7 +177,7 @@ public static double[] NextDoubles(this Random random, int length) var prngs = new Random[dimension]; for (int i = 0; i < dimension; i++) { - prngs[i] = random.GetType() == typeof(MersenneTwister) ? new MersenneTwister(random.Next()) : new Random(random.Next()); + prngs[i] = random is MersenneTwister ? new MersenneTwister(random.Next()) : new Random(random.Next()); for (int j = 0; j < length; j++) { values[j, i] = prngs[i].NextDouble(); diff --git a/Numerics/Utilities/JsonConverters.cs b/Numerics/Utilities/JsonConverters.cs index cc079278..168bc583 100644 --- a/Numerics/Utilities/JsonConverters.cs +++ b/Numerics/Utilities/JsonConverters.cs @@ -29,8 +29,10 @@ */ using System; +using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; +using Numerics.Data.Statistics; using Numerics.Distributions; namespace Numerics.Utilities @@ -75,17 +77,17 @@ public class Double2DArrayConverter : JsonConverter /// /// Expects JSON in the format: { "rows": int, "cols": int, "data": double[] } /// - public override double[,] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + public override double[,]? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType == JsonTokenType.Null) - return null!; + return null; if (reader.TokenType != JsonTokenType.StartObject) throw new JsonException("Expected StartObject token"); int rows = 0; int cols = 0; - double[]? data = null!; + double[]? data = null; while (reader.Read()) { @@ -115,14 +117,16 @@ public class Double2DArrayConverter : JsonConverter if (data == null || rows == 0 || cols == 0) return new double[0, 0]; + if (data.Length != rows * cols) + throw new System.Text.Json.JsonException($"Array dimension mismatch: expected {rows * cols} elements but found {data.Length}."); + double[,] result = new double[rows, cols]; int index = 0; for (int i = 0; i < rows; i++) { for (int j = 0; j < cols; j++) { - if (index < data.Length) - result[i, j] = data[index++]; + result[i, j] = data[index++]; } } @@ -214,17 +218,17 @@ public class String2DArrayConverter : JsonConverter /// /// Expects JSON in the format: { "rows": int, "cols": int, "data": string[] } /// - public override string[,] Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + public override string[,]? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType == JsonTokenType.Null) - return null!; + return null; if (reader.TokenType != JsonTokenType.StartObject) throw new JsonException("Expected StartObject token"); int rows = 0; int cols = 0; - string[]? data = null!; + string[]? data = null; while (reader.Read()) { @@ -368,16 +372,16 @@ public class UnivariateDistributionConverter : JsonConverter /// - public override UnivariateDistributionBase Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + public override UnivariateDistributionBase? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { if (reader.TokenType == JsonTokenType.Null) - return null!; + return null; if (reader.TokenType != JsonTokenType.StartObject) throw new JsonException("Expected StartObject token"); UnivariateDistributionType? distributionType = null; - double[]? parameters = null!; + double[]? parameters = null; while (reader.Read()) { @@ -408,11 +412,11 @@ public override UnivariateDistributionBase Read(ref Utf8JsonReader reader, Type try { var distribution = UnivariateDistributionFactory.CreateDistribution(distributionType.Value); - if (distribution != null! && parameters != null && parameters.Length > 0) + if (distribution is not null && parameters != null && parameters.Length > 0) { distribution.SetParameters(parameters); } - return distribution!; + return distribution; } catch { @@ -443,7 +447,7 @@ public override UnivariateDistributionBase Read(ref Utf8JsonReader reader, Type /// public override void Write(Utf8JsonWriter writer, UnivariateDistributionBase value, JsonSerializerOptions options) { - if (value == null!) + if (value is null) { writer.WriteNullValue(); return; @@ -471,4 +475,193 @@ public override void Write(Utf8JsonWriter writer, UnivariateDistributionBase val writer.WriteEndObject(); } } + + /// + /// Custom JSON converter for . + /// Serializes and deserializes histogram data without modifying the Histogram class's public API. + /// + /// + /// + /// This converter serializes a Histogram into a JSON object with its scalar properties + /// and a "Bins" array containing each bin's LowerBound, UpperBound, and Frequency. + /// + /// + /// JSON Format: + /// + /// { + /// "LowerBound": 0.0, + /// "UpperBound": 10.0, + /// "NumberOfBins": 5, + /// "BinWidth": 2.0, + /// "Bins": [ + /// { "LowerBound": 0.0, "UpperBound": 2.0, "Frequency": 3 }, + /// ... + /// ] + /// } + /// + /// + /// + public class HistogramConverter : JsonConverter + { + /// + /// Reads and converts JSON to a instance. + /// + /// The JSON reader. + /// The type to convert. + /// Serialization options. + /// + /// A Histogram reconstructed from the JSON, or null if the JSON is null. + /// + /// + /// Thrown when the JSON format is invalid (e.g., missing StartObject token). + /// + public override Histogram? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return null; + + if (reader.TokenType != JsonTokenType.StartObject) + throw new JsonException("Expected StartObject token for Histogram."); + + double lowerBound = 0; + double upperBound = 0; + int numberOfBins = 0; + double binWidth = 0; + var bins = new List(); + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName) + { + case "LowerBound": + lowerBound = reader.GetDouble(); + break; + case "UpperBound": + upperBound = reader.GetDouble(); + break; + case "NumberOfBins": + numberOfBins = reader.GetInt32(); + break; + case "BinWidth": + binWidth = reader.GetDouble(); + break; + case "Bins": + bins = ReadBins(ref reader); + break; + default: + // Skip unknown properties (e.g., DataCount, Mean, Median, Mode, + // StandardDeviation from old serialization format without converter) + reader.Skip(); + break; + } + } + } + + if (bins.Count == 0) + return null; + + return new Histogram(lowerBound, upperBound, numberOfBins, binWidth, bins); + } + + /// + /// Reads an array of histogram bins from JSON. + /// + /// The JSON reader positioned at the start of the array. + /// A list of histogram bins. + private static List ReadBins(ref Utf8JsonReader reader) + { + var bins = new List(); + + if (reader.TokenType != JsonTokenType.StartArray) + return bins; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + break; + + if (reader.TokenType == JsonTokenType.StartObject) + { + double lower = 0, upper = 0; + int frequency = 0; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + break; + + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? name = reader.GetString(); + reader.Read(); + + switch (name) + { + case "LowerBound": + lower = reader.GetDouble(); + break; + case "UpperBound": + upper = reader.GetDouble(); + break; + case "Frequency": + frequency = reader.GetInt32(); + break; + default: + reader.Skip(); + break; + } + } + } + + bins.Add(new Histogram.Bin(lower, upper, frequency)); + } + } + + return bins; + } + + /// + /// Writes a as JSON. + /// + /// The JSON writer. + /// The histogram to serialize. + /// Serialization options. + public override void Write(Utf8JsonWriter writer, Histogram value, JsonSerializerOptions options) + { + if (value == null) + { + writer.WriteNullValue(); + return; + } + + writer.WriteStartObject(); + writer.WriteNumber("LowerBound", value.LowerBound); + writer.WriteNumber("UpperBound", value.UpperBound); + writer.WriteNumber("NumberOfBins", value.NumberOfBins); + writer.WriteNumber("BinWidth", value.BinWidth); + + writer.WritePropertyName("Bins"); + writer.WriteStartArray(); + for (int i = 0; i < value.NumberOfBins; i++) + { + var bin = value[i]; + writer.WriteStartObject(); + writer.WriteNumber("LowerBound", bin.LowerBound); + writer.WriteNumber("UpperBound", bin.UpperBound); + writer.WriteNumber("Frequency", bin.Frequency); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + + writer.WriteEndObject(); + } + } } \ No newline at end of file diff --git a/Numerics/Utilities/SafeProgressReporter.cs b/Numerics/Utilities/SafeProgressReporter.cs index b2817074..964c2029 100644 --- a/Numerics/Utilities/SafeProgressReporter.cs +++ b/Numerics/Utilities/SafeProgressReporter.cs @@ -79,11 +79,20 @@ public SafeProgressReporter(string taskName) private double _previousProgress = -0.0000000000001d; private string _previousMessage = ""; private MessageType _previousMessageType = MessageType.Status; - private Process _externalProcess = null!; + private Process? _externalProcess; private List _subProgReporterCollection = new List(); private CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource(); + /// + /// Callback for invoking progress event handlers on the synchronization context. + /// protected readonly SendOrPostCallback _invokeProgressHandlers; + /// + /// Callback for invoking message event handlers on the synchronization context. + /// protected readonly SendOrPostCallback _invokeMessageHandlers; + /// + /// The synchronization context used for marshaling events to the UI thread. + /// protected SynchronizationContext? _synchronizationContext; /// @@ -99,7 +108,12 @@ public SafeProgressReporter(string taskName) /// /// Returns the message count. /// - public int MessageCount { get; private set; } + private int _messageCount; + + /// + /// Returns the message count. + /// + public int MessageCount => _messageCount; /// /// Returns the task name. @@ -114,7 +128,7 @@ public SafeProgressReporter(string taskName) /// /// The external process being executed. /// - protected Process ExternalProcess => _externalProcess; + protected Process? ExternalProcess => _externalProcess; /// /// Determines if cancellation was requested. @@ -132,7 +146,7 @@ public ReadOnlyCollection ChildReporters /// /// Event is raised when the progress is reported. /// - public event ProgressReportedEventHandler ProgressReported = null!; + public event ProgressReportedEventHandler? ProgressReported; /// /// Delegate for handling progress reported events. @@ -145,7 +159,7 @@ public ReadOnlyCollection ChildReporters /// /// Event is raised when a message is reported. /// - public event MessageReportedEventHandler MessageReported = null!; + public event MessageReportedEventHandler? MessageReported; /// /// Delegate for handling message reported events. @@ -156,7 +170,7 @@ public ReadOnlyCollection ChildReporters /// /// Event is raised when the task starts. /// - public event TaskStartedEventHandler TaskStarted = null!; + public event TaskStartedEventHandler? TaskStarted; /// /// Delegate for handling task started events. @@ -166,7 +180,7 @@ public ReadOnlyCollection ChildReporters /// /// Event is raised when the task ended. /// - public event TaskEndedEventHandler TaskEnded = null!; + public event TaskEndedEventHandler? TaskEnded; /// /// Delegate for handling task ended events. @@ -176,7 +190,7 @@ public ReadOnlyCollection ChildReporters /// /// Event is raised when a child reporter is created. /// - public event ChildReporterCreatedEventHandler ChildReporterCreated = null!; + public event ChildReporterCreatedEventHandler? ChildReporterCreated; /// /// Delegate for handling child reporter created events. @@ -189,9 +203,21 @@ public ReadOnlyCollection ChildReporters /// public enum MessageType { + /// + /// A status message. + /// Status, + /// + /// A success message. + /// Success, + /// + /// A warning message. + /// Warning, + /// + /// A fatal error message. + /// FatalError } @@ -200,9 +226,24 @@ public enum MessageType /// public struct MessageContentStruct { + /// + /// The message text. + /// public string Message; + /// + /// The type of message. + /// public MessageType MessageType; + /// + /// The reporter that generated the message. + /// public SafeProgressReporter Reporter; + /// + /// Creates a new message content structure. + /// + /// The message text. + /// The type of message. + /// The reporter that generated the message. public MessageContentStruct(string message, MessageType messageType, SafeProgressReporter reporter) { Message = message; @@ -219,7 +260,7 @@ public MessageContentStruct(string message, MessageType messageType, SafeProgres /// Set synchronization context. /// /// The context. - protected void SetContext(SynchronizationContext context) + protected void SetContext(SynchronizationContext? context) { _synchronizationContext = context; } @@ -290,7 +331,7 @@ public void ReportError(string message) private void InvokeProgressHandlers(object? state) { double prog = ((double[])state!)[0]; - double prevProg = ((double[])state)[1]; + double prevProg = ((double[])state!)[1]; if (prevProg < 0d) prevProg = 0d; OnProgressReported(prog); @@ -353,7 +394,7 @@ public SafeProgressReporter CreateProgressModifier(float fractionOfTotal, string if (string.IsNullOrEmpty(subTaskName)) subTaskName = TaskName; var child = new SafeProgressReporter(subTaskName); - child.SetContext(_synchronizationContext!); + child.SetContext(_synchronizationContext); child._previousProgress = 0d; child.ProgressReported += (reporter, prog, progDelta) => ReportProgress(_previousProgress + progDelta * fractionOfTotal); child.MessageReported += msg => ReportMessage(msg); @@ -395,7 +436,7 @@ public void ReportProgress(double progress) public void ReportMessage(string message, MessageType messageType = MessageType.Status) { _synchronizationContext?.Post(_invokeMessageHandlers, new MessageContentStruct(message, messageType, this)); - MessageCount += 1; + System.Threading.Interlocked.Increment(ref _messageCount); _previousMessage = message; _previousMessageType = messageType; } @@ -407,7 +448,7 @@ public void ReportMessage(string message, MessageType messageType = MessageType. protected void ReportMessage(MessageContentStruct message) { _synchronizationContext?.Post(_invokeMessageHandlers, message); - MessageCount += 1; + System.Threading.Interlocked.Increment(ref _messageCount); _previousMessage = message.Message; _previousMessageType = message.MessageType; } diff --git a/Numerics/Utilities/Tools.cs b/Numerics/Utilities/Tools.cs index 5cad3c99..a622098e 100644 --- a/Numerics/Utilities/Tools.cs +++ b/Numerics/Utilities/Tools.cs @@ -781,7 +781,7 @@ public static double[] Sequence(double start, double end, double step = 1) /// Returns a compressed a byte array. /// /// An array of bytes. - public static byte[] Compress(byte[] data) + public static byte[]? Compress(byte[]? data) { if (data is null) return null!; var output = new MemoryStream(); @@ -796,7 +796,7 @@ public static byte[] Compress(byte[] data) /// Returns a decompressed byte array. /// /// An array of bytes. - public static byte[] Decompress(byte[] data) + public static byte[]? Decompress(byte[]? data) { if (data is null) return null!; var input = new MemoryStream(data); diff --git a/README.md b/README.md index ae83967c..3433b79e 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,48 @@ # Numerics -***Numerics*** is a free and open-source library for .NET developed by the U.S. Army Corps of Engineers Risk Management Center (USACE-RMC). ***Numerics*** provides a comprehensive set of methods and algorithms for numerical computations and statistical analysis. The library includes routines for interpolation, regression, time series data, statistics, machine learning, probability distributions, bootstrap uncertainty analysis, Bayesian Markov Chain Monte Carlo, optimization, root finding, and more. + +[![CI](https://github.com/USACE-RMC/Numerics/actions/workflows/Integration.yml/badge.svg)](https://github.com/USACE-RMC/Numerics/actions/workflows/Integration.yml) +[![NuGet](https://img.shields.io/nuget/v/RMC.Numerics)](https://www.nuget.org/packages/RMC.Numerics/) +[![License: BSD-3-Clause](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](LICENSE) + +***Numerics*** is a free and open-source numerical computing library for .NET developed by the U.S. Army Corps of Engineers Risk Management Center (USACE-RMC). It provides methods and algorithms for probability distributions, statistical analysis, numerical methods, optimization, machine learning, and Bayesian MCMC sampling — with a focus on hydrological and risk assessment applications. + +## Supported Frameworks + +| Framework | Version | +|-----------|---------| +| .NET | 10.0, 9.0, 8.0 | +| .NET Framework | 4.8.1 | + +Install via NuGet: +``` +dotnet add package RMC.Numerics +``` +Or search for [RMC.Numerics](https://www.nuget.org/packages/RMC.Numerics/) in the NuGet Package Manager. ## Documentation -📚 **[User Guide and API Documentation](docs/index.md)** - Comprehensive documentation with examples and mathematical explanations +**[User Guide and API Documentation](docs/index.md)** — Comprehensive documentation with code examples and mathematical explanations. -The documentation covers: -- **Distributions**: Univariate probability distributions, parameter estimation, uncertainty analysis, copulas -- **Statistics**: Descriptive statistics, goodness-of-fit metrics, hypothesis tests -- **Mathematics**: Numerical integration, differentiation, optimization, linear algebra, root finding -- **Data**: Interpolation methods, time series analysis -- **Sampling**: Random number generation, MCMC methods including RWMH, ARWMH, DE-MCz, HMC, and Gibbs sampling +| Section | Topics | +|---------|--------| +| [Mathematics](docs/mathematics/integration.md) | Integration, differentiation, optimization, root finding, linear algebra, ODE solvers, special functions | +| [Data](docs/data/interpolation.md) | Interpolation, linear regression, time series analysis | +| [Statistics](docs/statistics/descriptive.md) | Descriptive statistics, goodness-of-fit metrics, hypothesis tests | +| [Distributions](docs/distributions/univariate.md) | 40+ univariate distributions, parameter estimation, uncertainty analysis, copulas, multivariate distributions | +| [Machine Learning](docs/machine-learning/machine-learning.md) | GLM, decision trees, random forests, KNN, naive Bayes, k-means, GMM | +| [Sampling](docs/sampling/mcmc.md) | MCMC (RWMH, ARWMH, DE-MCz, HMC, NUTS, Gibbs), random generation, convergence diagnostics | +| [References](docs/references.md) | Consolidated bibliography | ## Support -The RMC is committed to maintaining and supporting the software, providing regular updates, bug fixes, and enhancements on an annual basis or as needed. -The repository contains a unit testing library with more than 1,000 tests. These tests also provide examples for using the classes and methods in the library. +USACE-RMC is committed to maintaining and supporting the library with regular updates, bug fixes, and enhancements. + +The repository includes a unit testing library with over 1,000 tests that also serve as usage examples for the classes and methods in the library. + +## Contributing + +Contributions are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on bug reports, feature requests, and pull requests. -## Installation Instructions -We recommend using [NuGet](https://www.nuget.org/) for convenient installation of the [RMC.Numerics](https://www.nuget.org/packages/RMC.Numerics/) package. +## License +See [LICENSE](LICENSE) for details. diff --git a/Test_Numerics/AssemblyAttributes.cs b/Test_Numerics/AssemblyAttributes.cs new file mode 100644 index 00000000..3119b2d6 --- /dev/null +++ b/Test_Numerics/AssemblyAttributes.cs @@ -0,0 +1,3 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: DoNotParallelize] diff --git a/Test_Numerics/Data/Interpolation/Test_Bilinear.cs b/Test_Numerics/Data/Interpolation/Test_Bilinear.cs index 540ff239..345db3b4 100644 --- a/Test_Numerics/Data/Interpolation/Test_Bilinear.cs +++ b/Test_Numerics/Data/Interpolation/Test_Bilinear.cs @@ -93,7 +93,7 @@ public void Test_BiLinear() double x1 = 350d; double x2 = 75d; double y = bilinear.Interpolate(x1, x2); - Assert.AreEqual(874.84d, y, 1E-6); + Assert.AreEqual(874.84d, y, 1E-6); } /// @@ -136,7 +136,7 @@ public void Test_Log() var LogLinLin = new Bilinear(x1Array, x2Array, yArray) { X1Transform = Transform.Logarithmic }; double y1 = LogLinLin.Interpolate(x1, x2); - Assert.AreEqual(874.909523653025d, y1, 1E-6); + Assert.AreEqual(874.909523653025d, y1, 1E-6); var LinLogLin = new Bilinear(x1Array, x2Array, yArray) { X2Transform = Transform.Logarithmic }; double y2 = LinLogLin.Interpolate(x1, x2); @@ -144,7 +144,7 @@ public void Test_Log() var LinLinLog = new Bilinear(x1Array, x2Array, yArray) { YTransform = Transform.Logarithmic }; double y3 = LinLinLog.Interpolate(x1, x2); - Assert.AreEqual(874.8164, y3, 1E-4); + Assert.AreEqual(874.8164, y3, 1E-4); var LinLogLog = new Bilinear(x1Array, x2Array, yArray) { X2Transform = Transform.Logarithmic, YTransform = Transform.Logarithmic }; double y4 = LinLogLog.Interpolate(x1, x2); diff --git a/Test_Numerics/Data/Interpolation/Test_CubicSpline.cs b/Test_Numerics/Data/Interpolation/Test_CubicSpline.cs index 51faa09a..a7f6df50 100644 --- a/Test_Numerics/Data/Interpolation/Test_CubicSpline.cs +++ b/Test_Numerics/Data/Interpolation/Test_CubicSpline.cs @@ -66,7 +66,7 @@ public void Test_Sequential() values[i - 1] = i; var spline = new CubicSpline(values, values); var lo = spline.SequentialSearch(872.5d); - Assert.AreEqual(871,lo); + Assert.AreEqual(871, lo); Array.Reverse(values); var spline2 = new CubicSpline(values, values, SortOrder.Descending); @@ -90,7 +90,7 @@ public void Test_Bisection() Array.Reverse(values); var spline2 = new CubicSpline(values, values, SortOrder.Descending); lo = spline2.BisectionSearch(872.5); - Assert.AreEqual(127,lo); + Assert.AreEqual(127, lo); } /// @@ -104,7 +104,7 @@ public void Test_Hunt() values[i - 1] = i; var spline = new CubicSpline(values, values); var lo = spline.HuntSearch(872.5d); - Assert.AreEqual(871,lo); + Assert.AreEqual(871, lo); Array.Reverse(values); var spline2 = new CubicSpline(values, values, SortOrder.Descending); @@ -123,7 +123,7 @@ public void Test_Interpolate() var spline = new CubicSpline(XArray, YArray); double X = 8d; double Y = spline.Interpolate(X); - Assert.AreEqual(11.4049889205445d, Y, 1E-6); + Assert.AreEqual(11.4049889205445d, Y, 1E-6); } /// @@ -162,7 +162,7 @@ public void Test_Interpolate_List() var Y = spline.Interpolate(X); var true_Y = new double[] { 9.96, 11.4049889205445, 12.8430203387148, 14.2671367521368, 15.6703806584362, 17.045794555239, 18.3864209401709, 19.6853023108579, 20.9354811649256, 22.13, 23.2634521159782, 24.3366340218424, 25.3518930288462, 26.3115764482431, 27.2180315912868, 28.0736057692308, 28.8806462933286, 29.6415004748338, 30.358515625, 31.0340390550807, 31.6704180763295, 32.27, 32.8352193880579, 33.3688598053181, 33.8737920673077, 34.3528869895537, 34.8090153875831, 35.2450480769231, 35.6638558731007, 36.0683095916429, 36.4612800480769, 36.8456380579297, 37.2242544367284, 37.6}; for (int i = 0; i < X.Length; i++) - Assert.AreEqual(Y[i], true_Y[i], 1E-6); + Assert.AreEqual(true_Y[i], Y[i], 1E-6); } } diff --git a/Test_Numerics/Data/Interpolation/Test_Linear.cs b/Test_Numerics/Data/Interpolation/Test_Linear.cs index 646c28c3..881e0d60 100644 --- a/Test_Numerics/Data/Interpolation/Test_Linear.cs +++ b/Test_Numerics/Data/Interpolation/Test_Linear.cs @@ -65,12 +65,12 @@ public void Test_Sequential() values[i - 1] = i; var LI = new Linear(values, values); var lo = LI.SequentialSearch(872.5d); - Assert.AreEqual(871,lo); + Assert.AreEqual(871, lo); Array.Reverse(values); LI = new Linear(values, values, SortOrder.Descending); lo = LI.SequentialSearch(872.5); - Assert.AreEqual(127,lo); + Assert.AreEqual(127, lo); } /// @@ -122,7 +122,7 @@ public void Test_Lin() var LI = new Linear(XArray, YArray); double X = 75d; double Y = LI.Interpolate(X); - Assert.AreEqual(150.0d, Y, 1E-6); + Assert.AreEqual(150.0d, Y, 1E-6); } /// @@ -155,15 +155,15 @@ public void Test_Log() var LinLog = new Linear(XArray, YArray) { YTransform = Transform.Logarithmic }; double Y1 = LinLog.Interpolate(X); - Assert.AreEqual(141.42135623731d, Y1, 1E-6); + Assert.AreEqual(141.42135623731d, Y1, 1E-6); var LogLin = new Linear(XArray, YArray) { XTransform = Transform.Logarithmic }; double Y2 = LogLin.Interpolate(X); - Assert.AreEqual(158.496250072116d, Y2, 1E-6); + Assert.AreEqual(158.496250072116d, Y2, 1E-6); ; var LogLog = new Linear(XArray, YArray) { XTransform = Transform.Logarithmic, YTransform = Transform.Logarithmic }; double Y3 = LogLog.Interpolate(X); - Assert.AreEqual(150.0d, Y3, 1E-6); + Assert.AreEqual(150.0d, Y3, 1E-6); } /// @@ -214,15 +214,15 @@ public void Test_Z() var LinZ = new Linear(XArray, YArray) { YTransform = Transform.NormalZ }; double Y1 = LinZ.Interpolate(X); - Assert.AreEqual(0.358762529d, Y1, 1E-6); + Assert.AreEqual(0.358762529d, Y1, 1E-6); var ZLin = new Linear(XArray, YArray) { XTransform = Transform.NormalZ }; double Y2 = ZLin.Interpolate(X); - Assert.AreEqual(0.362146174d, Y2, 1E-6); + Assert.AreEqual(0.362146174d, Y2, 1E-6); var ZZ = new Linear(XArray, YArray) { XTransform = Transform.NormalZ, YTransform = Transform.NormalZ }; double Y3 = ZZ.Interpolate(X); - Assert.AreEqual(0.36093855992815d, Y3, 1E-6); + Assert.AreEqual(0.36093855992815d, Y3, 1E-6); } /// @@ -274,7 +274,7 @@ public void Test_RevLin() var LI = new Linear(XArray, YArray, SortOrder.Descending); double X = 75d; double Y = LI.Interpolate(X); - Assert.AreEqual(150.0d, Y, 1E-6); + Assert.AreEqual(150.0d, Y, 1E-6); } /// @@ -292,11 +292,11 @@ public void Test_Rev_Log() var LinLog = new Linear(XArray, YArray, SortOrder.Descending) { YTransform = Transform.Logarithmic }; double Y1 = LinLog.Interpolate(X); - Assert.AreEqual(141.42135623731d, Y1, 1E-6); + Assert.AreEqual(141.42135623731d, Y1, 1E-6); var LogLin = new Linear(XArray, YArray, SortOrder.Descending) { XTransform = Transform.Logarithmic }; double Y2 = LogLin.Interpolate(X); - Assert.AreEqual(158.496250072116d, Y2, 1E-6); + Assert.AreEqual(158.496250072116d, Y2, 1E-6); var LogLog = new Linear(XArray, YArray, SortOrder.Descending) { XTransform = Transform.Logarithmic, YTransform = Transform.Logarithmic }; double Y3 = LogLog.Interpolate(X); @@ -347,7 +347,7 @@ public void Test_Lin_List() var yVals = LI.Interpolate(xVals); var trueVals = new double[] { 100, 128.888888888889, 177.777777777778, 226.666666666667, 275.555555555556, 324.444444444444, 373.333333333333, 422.222222222222, 471.111111111111, 500 }; for (int i = 1; i < N; i++) - Assert.AreEqual(yVals[i], trueVals[i], 1E-6); + Assert.AreEqual(trueVals[i], yVals[i], 1E-6); } /// diff --git a/Test_Numerics/Data/Interpolation/Test_Polynomial.cs b/Test_Numerics/Data/Interpolation/Test_Polynomial.cs index be9e710e..0a2bcbf1 100644 --- a/Test_Numerics/Data/Interpolation/Test_Polynomial.cs +++ b/Test_Numerics/Data/Interpolation/Test_Polynomial.cs @@ -66,12 +66,12 @@ public void Test_Sequential() values[i - 1] = i; var poly = new Polynomial(3, values, values); var lo = poly.SequentialSearch(872.5d); - Assert.AreEqual(871, lo ); + Assert.AreEqual(871, lo); Array.Reverse(values); var poly2 = new Polynomial(3, values, values, SortOrder.Descending); lo = poly2.SequentialSearch(872.5); - Assert.AreEqual(127,lo); + Assert.AreEqual(127, lo); } /// @@ -85,12 +85,12 @@ public void Test_Bisection() values[i - 1] = i; var poly = new Polynomial(3, values, values); var lo = poly.BisectionSearch(872.5d); - Assert.AreEqual(871,lo); + Assert.AreEqual(871, lo); Array.Reverse(values); var poly2 = new Polynomial(3, values, values, SortOrder.Descending); lo = poly2.BisectionSearch(872.5); - Assert.AreEqual(127,lo); + Assert.AreEqual(127, lo); } /// @@ -104,12 +104,12 @@ public void Test_Hunt() values[i - 1] = i; var poly = new Polynomial(3, values, values); var lo = poly.HuntSearch(872.5d); - Assert.AreEqual(871,lo); + Assert.AreEqual(871, lo); Array.Reverse(values); var poly2 = new Polynomial(3, values, values, SortOrder.Descending); lo = poly2.HuntSearch(872.5); - Assert.AreEqual(127,lo); + Assert.AreEqual(127, lo); } /// @@ -123,7 +123,7 @@ public void Test_Interpolate_Order3() var poly = new Polynomial(3, XArray, YArray); double X = 8d; double Y = poly.Interpolate(X); - Assert.AreEqual(11.5415808882467, Y, 1E-6); + Assert.AreEqual(11.5415808882467, Y, 1E-6); } /// @@ -162,7 +162,7 @@ public void Test_Interpolate_List() var Y = poly.Interpolate(X); var true_Y = new double[] { 9.95999999999906, 11.5415808882467, 13.0626606341182, 14.5245941558435, 15.9287363716526, 17.2764421997752, 18.5690665584413, 19.807964365881, 20.994490540324, 22.1300000000003, 23.2158476631398, 24.2533884479724, 25.2439772727281, 26.1889690556368, 27.0897187149283, 27.9475811688326, 28.7639113355797, 29.5400641333994, 30.2773944805217, 30.9772572951765, 31.6410074955936, 32.2700000000031, 32.8655897266348, 33.4291315937186, 33.9619805194845, 34.4654914221624, 34.9410192199823, 35.3899188311739, 35.8135451739673, 36.2132531665923, 36.5903977272789, 36.9463337742571, 37.2824162257566, 37.6000000000075, 37.9004400152396, 38.1850911896829, 38.4553084415673, 38.7124466891227, 38.9578608505791, 39.1929058441663 }; for (int i = 0; i < X.Length; i++) - Assert.AreEqual(Y[i], true_Y[i], 1E-6); + Assert.AreEqual(true_Y[i], Y[i], 1E-6); } } } diff --git a/Test_Numerics/Data/Paired Data/Test_Ordinate.cs b/Test_Numerics/Data/Paired Data/Test_Ordinate.cs index a3437a3a..4c7c90a9 100644 --- a/Test_Numerics/Data/Paired Data/Test_Ordinate.cs +++ b/Test_Numerics/Data/Paired Data/Test_Ordinate.cs @@ -71,11 +71,11 @@ public void Test_Construction() var ordinate4 = new Ordinate(double.NaN, 4); Assert.AreEqual(ordinate1, ordinate2); - Assert.AreEqual(2,ordinate1.X); + Assert.AreEqual(2, ordinate1.X); Assert.AreEqual(4, ordinate1.Y); Assert.AreEqual(2, ordinate2.X); Assert.AreEqual(4, ordinate2.Y); - + Assert.IsTrue(ordinate1.IsValid); Assert.IsFalse(ordinate3.IsValid); Assert.IsFalse(ordinate4.IsValid); diff --git a/Test_Numerics/Data/Paired Data/Test_PairedDataInterpolation.cs b/Test_Numerics/Data/Paired Data/Test_PairedDataInterpolation.cs index 5ff08e81..f9fe4ab8 100644 --- a/Test_Numerics/Data/Paired Data/Test_PairedDataInterpolation.cs +++ b/Test_Numerics/Data/Paired Data/Test_PairedDataInterpolation.cs @@ -56,7 +56,7 @@ public void Test_Sequential() opd.Add(new Ordinate(i, i)); // X var lo = opd.SequentialSearchX(872.5d); - Assert.AreEqual(871,lo); + Assert.AreEqual(871, lo); // Y lo = opd.SequentialSearchY(872.5d); Assert.AreEqual(871, lo); @@ -147,7 +147,7 @@ public void Test_Lin() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Ascending, true, SortOrder.Ascending); double X = 75d; double Y = opd.GetYFromX(X); - Assert.AreEqual(150.0d, Y, 1E-6); + Assert.AreEqual(150.0d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y); @@ -168,7 +168,7 @@ public void Test_LinLog() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Ascending, true, SortOrder.Ascending); double X = 75d; double Y = opd.GetYFromX(X, Transform.None, Transform.Logarithmic); - Assert.AreEqual(141.42135623731d, Y, 1E-6); + Assert.AreEqual(141.42135623731d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y, Transform.None, Transform.Logarithmic); @@ -188,7 +188,7 @@ public void Test_LogLin() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Ascending, true, SortOrder.Ascending); double X = 75d; double Y = opd.GetYFromX(X, Transform.Logarithmic, Transform.None); - Assert.AreEqual(158.496250072116d, Y, 1E-6); + Assert.AreEqual(158.496250072116d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y, Transform.Logarithmic, Transform.None); @@ -208,7 +208,7 @@ public void Test_LogLog() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Ascending, true, SortOrder.Ascending); double X = 75d; double Y = opd.GetYFromX(X, Transform.Logarithmic, Transform.Logarithmic); - Assert.AreEqual(150.0d, Y, 1E-6); + Assert.AreEqual(150.0d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y, Transform.Logarithmic, Transform.Logarithmic); @@ -228,7 +228,7 @@ public void Test_LinZ() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Ascending, true, SortOrder.Ascending); double X = 0.18d; double Y = opd.GetYFromX(X, Transform.None, Transform.NormalZ); - Assert.AreEqual(0.358762529d, Y, 1E-6); + Assert.AreEqual(0.358762529d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y, Transform.None, Transform.NormalZ); @@ -248,7 +248,7 @@ public void Test_ZLin() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Ascending, true, SortOrder.Ascending); double X = 0.18d; double Y = opd.GetYFromX(X, Transform.NormalZ, Transform.None); - Assert.AreEqual(0.362146174d, Y, 1E-6); + Assert.AreEqual(0.362146174d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y, Transform.NormalZ, Transform.None); @@ -268,7 +268,7 @@ public void Test_ZZ() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Ascending, true, SortOrder.Ascending); double X = 0.18d; double Y = opd.GetYFromX(X, Transform.NormalZ, Transform.NormalZ); - Assert.AreEqual(0.36093855992815d, Y, 1E-6); + Assert.AreEqual(0.36093855992815d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y, Transform.NormalZ, Transform.NormalZ); @@ -290,7 +290,7 @@ public void Test_RevLinear() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Descending, true, SortOrder.Descending); double X = 75d; double Y = opd.GetYFromX(X); - Assert.AreEqual(150.0d, Y, 1E-6); + Assert.AreEqual(150.0d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y); @@ -312,7 +312,7 @@ public void Test_RevLinLog() var opd = new OrderedPairedData(XArray, YArray, true, SortOrder.Descending, true, SortOrder.Descending); double X = 75d; double Y = opd.GetYFromX(X, Transform.None, Transform.Logarithmic); - Assert.AreEqual(141.42135623731d, Y, 1E-6); + Assert.AreEqual(141.42135623731d, Y, 1E-6); // Given Y var xFromY = opd.GetXFromY(Y, Transform.None, Transform.Logarithmic); diff --git a/Test_Numerics/Data/Statistics/Test_BoxCox.cs b/Test_Numerics/Data/Statistics/Test_BoxCox.cs index 2bcfbfa8..72e54d2c 100644 --- a/Test_Numerics/Data/Statistics/Test_BoxCox.cs +++ b/Test_Numerics/Data/Statistics/Test_BoxCox.cs @@ -66,7 +66,7 @@ public void Test_Fit() var sample = new[] { 142.25d, 141.23d, 141.33d, 140.82d, 141.31d, 140.58d, 141.58d, 142.15d, 143.07d, 142.85d, 143.17d, 142.54d, 143.07d, 142.26d, 142.97d, 143.86d, 142.57d, 142.19d, 142.35d, 142.63d, 144.15d, 144.73d, 144.7d, 144.97d, 145.12d, 144.78d, 145.06d, 143.94d, 143.77d, 144.8d, 145.67d, 145.44d, 145.56d, 145.61d, 146.05d, 145.74d, 145.83d, 143.88d, 140.39d, 139.34d, 140.05d, 137.93d, 138.78d, 139.59d, 140.54d, 141.31d, 140.42d, 140.18d, 138.43d, 138.97d, 139.31d, 139.26d, 140.08d, 141.1d, 143.48d, 143.28d, 143.5d, 143.12d, 142.14d, 142.54d, 142.24d, 142.16d, 142.97d, 143.69d, 143.67d, 144.65d, 144.33d, 144.82d, 143.74d, 144.9d, 145.83d, 146.97d, 146.6d, 146.55d, 148.22d, 148.37d, 148.23d, 148.73d, 149.49d, 149.09d, 149.64d, 148.42d, 148.9d, 149.97d, 150.75d, 150.88d, 150.58d, 150.64d, 150.73d, 149.75d, 150.86d, 150.7d, 150.8d, 151.38d, 152.01d, 152.58d, 152.7d, 152.95d, 152.53d, 151.5d, 151.94d, 151.46d, 153.67d, 153.88d, 153.54d, 153.74d, 152.86d, 151.56d, 149.58d, 150.93d, 150.67d, 150.5d, 152.06d, 153.14d, 153.38d, 152.55d, 153.58d, 151.08d, 151.52d, 150.24d, 150.21d, 148.13d, 150.38d, 150.9d, 150.87d, 152.18d, 152.4d, 152.38d, 153.16d, 152.29d, 150.75d, 152.37d, 154.57d, 154.99d, 154.93d, 154.23d, 155.2d, 154.89d, 154.18d, 153.12d, 152.02d, 150.19d, 148.21d, 145.93d, 148.33d, 145.18d, 146.76d, 147.28d, 144.21d, 145.94d, 148.41d, 147.43d, 144.39d, 146.5d, 145.7d, 142.72d, 139.79d, 145.5d, 145.17d, 144.6d, 146.01d, 147.34d, 146.48d, 147.85d, 146.16d, 144.37d, 145.45d, 147.65d, 147.45d, 148.2d, 147.95d, 146.48d, 146.52d, 146.24d, 147.29d, 148.55d, 147.96d, 148.31d, 148.83d, 153.41d, 153.34d, 152.71d, 152.42d, 150.81d, 152.25d, 152.91d, 152.85d, 152.6d, 154.61d, 153.81d, 154.11d, 155.03d, 155.39d, 155.6d, 156.04d, 156.93d, 155.46d, 156.27d, 154.41d, 154.98d }; double l1 = 0d; BoxCox.FitLambda(sample, out l1); - Assert.AreEqual(1.670035d, l1, 1E-4); + Assert.AreEqual(1.670035d, l1, 1E-4); } /// diff --git a/Test_Numerics/Data/Statistics/Test_HypothesisTests.cs b/Test_Numerics/Data/Statistics/Test_HypothesisTests.cs index bd37842b..28a20ddd 100644 --- a/Test_Numerics/Data/Statistics/Test_HypothesisTests.cs +++ b/Test_Numerics/Data/Statistics/Test_HypothesisTests.cs @@ -73,14 +73,14 @@ public void Test_OneSampleTtest() var data = new double[] { 8.782932, 10.64199, -1.63955, -6.802458, 9.088312, -27.26934, 9.451478, -4.142762, -4.262396, -13.78983, -1.743717, 27.259681, 5.559418, 7.803247, -11.25798, 12.253498, -13.295363, -4.973664, 16.81069, 4.480855, 11.694329, 21.836776, -9.664926, -23.297061, -23.965643, 27.076463, -7.22471, 9.305697, 9.181852, -2.434665 }; var p = HypothesisTests.OneSampleTtest(data); double true_p = 0.6489; - Assert.AreEqual(p, true_p, 1E-4); + Assert.AreEqual(true_p, p, 1E-4); p = HypothesisTests.OneSampleTtest(data, 10); true_p = 0.001823; - Assert.AreEqual(p, true_p, 1E-4); + Assert.AreEqual(true_p, p, 1E-4); var t = HypothesisTests.OneSampleTtest(new double[] { 23, 15, -5, 7, 1, -10, 12, -8, 20, 8, -2, -5 }); - Assert.AreEqual(0.087585 * 2, t, 1E-6); + Assert.AreEqual(0.087585 * 2, t, 1E-6); } /// @@ -94,7 +94,7 @@ public void Test_EqualVarianceTtest() var p = HypothesisTests.EqualVarianceTtest(data1, data2); double true_p = 0.0185; - Assert.AreEqual(p, true_p, 1E-4); + Assert.AreEqual(true_p, p, 1E-4); } /// @@ -108,7 +108,7 @@ public void Test_UnequalVarianceTtest() var p = HypothesisTests.UnequalVarianceTtest(data1, data2); double true_p = 0.02043; - Assert.AreEqual(p, true_p, 1E-4); + Assert.AreEqual(true_p, p, 1E-4); } /// @@ -122,7 +122,7 @@ public void Test_PairedTtest() var p = HypothesisTests.PairedTtest(data1, data2); double true_p = 6.2E-9; - Assert.AreEqual(p, true_p, 1E-4); + Assert.AreEqual(true_p, p, 1E-4); } /// @@ -136,7 +136,7 @@ public void Test_Ftest() var p = HypothesisTests.Ftest(data1, data2); double true_p = 0.1825; - Assert.AreEqual(p, true_p, 1E-4); + Assert.AreEqual(true_p, p, 1E-4); } /// @@ -183,7 +183,7 @@ public void Test_JarqueBera() // R is different because it estimates moments differently. var p = HypothesisTests.JarqueBeraTest(data); double true_p = 3.444E-05 / 2; - Assert.AreEqual(p, true_p, 1E-5); + Assert.AreEqual(true_p, p, 1E-5); // known example var JB = HypothesisTests.JarqueBeraTest(new double[] { 4, 5, 5, 6, 9, 12, 13, 14, 14, 19, 22, 24, 25 }); @@ -202,7 +202,7 @@ public void Test_WaldWolfowitz() var p = HypothesisTests.WaldWolfowitzTest(data); double true_p = (1 - Normal.StandardCDF(1.167)) * 2d; // See page 5 of reference. - Assert.AreEqual(p, true_p, 1E-3); + Assert.AreEqual(true_p, p, 1E-3); } /// @@ -215,11 +215,11 @@ public void Test_LjungBox() var p = HypothesisTests.LjungBoxTest(data,5); double true_p = 0.2314; - Assert.AreEqual(p, true_p, 1E-4); + Assert.AreEqual(true_p, p, 1E-4); var p2 = HypothesisTests.LjungBoxTest(data, 30); var true_p2 = 0.7548; - Assert.AreEqual(p2, true_p2, 1E-4); + Assert.AreEqual(true_p2, p2, 1E-4); } @@ -235,7 +235,7 @@ public void Test_MannWhitney() var p = HypothesisTests.MannWhitneyTest(data2, data1); double true_p = (1 - Normal.StandardCDF(0.54)) * 2d; // See page 7 of reference. - Assert.AreEqual(p, true_p, 1E-2); + Assert.AreEqual(true_p, p, 1E-2); } /// @@ -252,8 +252,8 @@ public void Test_MannKendall() var data = new double[] { 122d, 244d, 214d, 173d, 229d, 156d, 212d, 263d, 146d, 183d, 161d, 205d, 135d, 331d, 225d, 174d, 98.8d, 149d, 238d, 262d, 132d, 235d, 216d, 240d, 230d, 192d, 195d, 172d, 173d, 172d, 153d, 142d, 317d, 161d, 201d, 204d, 194d, 164d, 183d, 161d, 167d, 179d, 185d, 117d, 192d, 337d, 125d, 166d, 99.1d, 202d, 230d, 158d, 262d, 154d, 164d, 182d, 164d, 183d, 171d, 250d, 184d, 205d, 237d, 177d, 239d, 187d, 180d, 173d, 174d }; var p = HypothesisTests.MannKendallTest(data); - double true_p = 0.7757; - Assert.AreEqual(p, true_p, 1E-4); + double true_p = 0.7757; + Assert.AreEqual(true_p, p, 1E-4); } /// @@ -287,8 +287,8 @@ public void Test_GrubbsBeck() MultipleGrubbsBeckTest.GrubbsBeckTest(data, out xHi, out xLo); double true_xHi = 378.2, true_xLo = 91.2; // page 9 of reference. - Assert.AreEqual(xHi, true_xHi, 1E-1); - Assert.AreEqual(xLo, true_xLo, 1E-1); + Assert.AreEqual(true_xHi, xHi, 1E-1); + Assert.AreEqual(true_xLo, xLo, 1E-1); } /// @@ -320,21 +320,21 @@ public void Test_MultipleGrubbsBeck() int Samp_True = 0; int LO; LO = MultipleGrubbsBeckTest.Function(BLU); - Assert.AreEqual(LO, BLU_True); + Assert.AreEqual(BLU_True, LO); LO = MultipleGrubbsBeckTest.Function(CGR); - Assert.AreEqual(LO, CGR_True); + Assert.AreEqual(CGR_True, LO); LO = MultipleGrubbsBeckTest.Function(FCK); - Assert.AreEqual(LO, FCK_True); + Assert.AreEqual(FCK_True, LO); LO = MultipleGrubbsBeckTest.Function(FOS); - Assert.AreEqual(LO, FOS_True); + Assert.AreEqual(FOS_True, LO); LO = MultipleGrubbsBeckTest.Function(GPV); - Assert.AreEqual(LO, GPV_True); + Assert.AreEqual(GPV_True, LO); LO = MultipleGrubbsBeckTest.Function(HCK); - Assert.AreEqual(LO, HCK_True); + Assert.AreEqual(HCK_True, LO); LO = MultipleGrubbsBeckTest.Function(LOP); - Assert.AreEqual(LO, LOP_True); + Assert.AreEqual(LOP_True, LO); LO = MultipleGrubbsBeckTest.Function(sample); - Assert.AreEqual(LO, Samp_True); + Assert.AreEqual(Samp_True, LO); } diff --git a/Test_Numerics/Data/Statistics/Test_Statistics.cs b/Test_Numerics/Data/Statistics/Test_Statistics.cs index 41b0f4dd..512f8869 100644 --- a/Test_Numerics/Data/Statistics/Test_Statistics.cs +++ b/Test_Numerics/Data/Statistics/Test_Statistics.cs @@ -70,7 +70,7 @@ public void Test_Minimum() { double val = Numerics.Data.Statistics.Statistics.Minimum(_sample1); double trueVal = 98.8d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -81,7 +81,7 @@ public void Test_Maximum() { double val = Numerics.Data.Statistics.Statistics.Maximum(_sample1); double trueVal = 337.0d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -92,7 +92,7 @@ public void Test_Mean() { double val = Numerics.Data.Statistics.Statistics.Mean(_sample1); double trueVal = 191.317391304348d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -108,7 +108,7 @@ public void Test_ParallelMean() double test = Numerics.Data.Statistics.Statistics.ParallelMean(_sample1); double regMean = Numerics.Data.Statistics.Statistics.Mean(_sample1); Assert.AreEqual(valid, test, 1E-6); - Assert.AreEqual(test, regMean, 1E-6); + Assert.AreEqual(regMean, test, 1E-6); } /// @@ -124,7 +124,7 @@ public void Test_GeometricMean() { double val = Numerics.Data.Statistics.Statistics.GeometricMean(_sample1); double trueVal = 185.685629284673d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -140,7 +140,7 @@ public void Test_HarmonicMean() { double val = Numerics.Data.Statistics.Statistics.HarmonicMean(_sample1); double trueVal = 180.183870381546d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -151,7 +151,7 @@ public void Test_Variance() { double val = Numerics.Data.Statistics.Statistics.Variance(_sample1); double trueVal = 2300.31616368286d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -163,7 +163,7 @@ public void Test_PopulationVariance() { double val = Numerics.Data.Statistics.Statistics.PopulationVariance(_sample1); double trueVal = 2266.97824826717d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -174,19 +174,19 @@ public void Test_StandardDeviation() { double val = Numerics.Data.Statistics.Statistics.StandardDeviation(_sample1); double trueVal = 47.9616113541118d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// - /// Test the PopulationStandardDeviation method against R by taking the square root of the population variance, calculated by multiplying the - /// "var()" method by (n-1) / n + /// Test the PopulationStandardDeviation method against R by taking the square root of the population variance, calculated by multiplying the + /// "var()" method by (n-1) / n /// [TestMethod] public void Test_PopulationStandardDeviation() { double val = Numerics.Data.Statistics.Statistics.PopulationStandardDeviation(_sample1); double trueVal = 47.6127950058298d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -246,7 +246,7 @@ public void Test_Skewness() { double val = Numerics.Data.Statistics.Statistics.Skewness(_sample1); double trueVal = 0.8605451107461d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -254,7 +254,7 @@ public void Test_Skewness() /// /// /// References: - /// original S, StatLib f, Leisch. bRTRpbF (2019). bootstrap: Functions for the Book "An Introduction to the Bootstrap". + /// original S, StatLib f, Leisch. bRTRpbF (2019). bootstrap: Functions for the Book "An Introduction to the Bootstrap". /// R package version 2019.6, /// [TestMethod] @@ -262,7 +262,7 @@ public void Test_JackKnifeStandardError() { double val = Numerics.Data.Statistics.Statistics.JackKnifeStandardError(_jackKnifeSample, Numerics.Data.Statistics.Statistics.Mean); double trueVal = 0.0372182162668589d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -297,7 +297,7 @@ public void Test_Kurtosis() { double val = Numerics.Data.Statistics.Statistics.Kurtosis(_sample1); double trueVal = 1.3434868130194d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -308,7 +308,7 @@ public void Test_Covariance() { double val = Numerics.Data.Statistics.Statistics.Covariance(_sample1, _sample2); double trueVal = -253.54405370844d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -319,7 +319,7 @@ public void Test_PopulationCovariance() { double val = Numerics.Data.Statistics.Statistics.PopulationCovariance(_sample1, _sample2); double trueVal = -249.869502205419d; - Assert.AreEqual(val, trueVal, 1E-10); + Assert.AreEqual(trueVal, val, 1E-10); } /// @@ -335,10 +335,10 @@ public void Test_ComputeProductMoments() double trueVal3 = 0.8605451107461d; double trueVal4 = 1.3434868130194d; - Assert.AreEqual(vals[0], trueVal1, 1E-10); - Assert.AreEqual(vals[1], trueVal2, 1E-10); - Assert.AreEqual(vals[2], trueVal3, 1E-10); - Assert.AreEqual(vals[3], trueVal4, 1E-10); + Assert.AreEqual(trueVal1, vals[0], 1E-10); + Assert.AreEqual(trueVal2, vals[1], 1E-10); + Assert.AreEqual(trueVal3, vals[2], 1E-10); + Assert.AreEqual(trueVal4, vals[3], 1E-10); } /// @@ -358,10 +358,10 @@ public void Test_ComputeLinearMoments() double trueVal3 = 0.1033903d; double trueVal4 = 0.1940943d; - Assert.AreEqual(lmoms[0], trueVal1, 1E-7); - Assert.AreEqual(lmoms[1], trueVal2, 1E-7); - Assert.AreEqual(lmoms[2], trueVal3, 1E-7); - Assert.AreEqual(lmoms[3], trueVal4, 1E-7); + Assert.AreEqual(trueVal1, lmoms[0], 1E-7); + Assert.AreEqual(trueVal2, lmoms[1], 1E-7); + Assert.AreEqual(trueVal3, lmoms[2], 1E-7); + Assert.AreEqual(trueVal4, lmoms[3], 1E-7); } /// @@ -372,39 +372,39 @@ public void Test_Percentiles() { double val0 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 0); double trueVal0 = 98.8d; - Assert.AreEqual(val0, trueVal0, 1E-2); + Assert.AreEqual(trueVal0, val0, 1E-2); double val1 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 0.01d); double trueVal1 = 99.004d; - Assert.AreEqual(val1, trueVal1, 1E-2); + Assert.AreEqual(trueVal1, val1, 1E-2); double val2 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 0.05d); double trueVal2 = 123.2d; - Assert.AreEqual(val2, trueVal2, 1E-2); + Assert.AreEqual(trueVal2, val2, 1E-2); double val3 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 0.25d); double trueVal3 = 164.0d; - Assert.AreEqual(val3, trueVal3, 1E-2); + Assert.AreEqual(trueVal3, val3, 1E-2); double val4 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 0.5d); double trueVal4 = 183.0d; - Assert.AreEqual(val4, trueVal4, 1E-2); + Assert.AreEqual(trueVal4, val4, 1E-2); double val5 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 0.75d); double trueVal5 = 216.0d; - Assert.AreEqual(val5, trueVal5, 1E-2); + Assert.AreEqual(trueVal5, val5, 1E-2); double val6 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 0.95d); double trueVal6 = 262.6d; - Assert.AreEqual(val6, trueVal6, 1E-2); + Assert.AreEqual(trueVal6, val6, 1E-2); double val7 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 0.99d); double trueVal7 = 332.92d; - Assert.AreEqual(val7, trueVal7, 1E-2); + Assert.AreEqual(trueVal7, val7, 1E-2); double val8 = Numerics.Data.Statistics.Statistics.Percentile(_sample1, 1); double trueVal8 = 337d; - Assert.AreEqual(val8, trueVal8, 1E-2); + Assert.AreEqual(trueVal8, val8, 1E-2); } /// @@ -506,5 +506,21 @@ public void Test_Entropy() double valid = 2.252937012209; Assert.AreEqual(valid, test, 1E-10); } + + /// + /// Verify HarmonicMean returns NaN when data contains zero, and correct value otherwise. + /// Reference: scipy.stats.hmean([1,2,3,4]) = 1.92 + /// + [TestMethod()] + public void Test_HarmonicMean_ZeroGuard() + { + // scipy.stats.hmean([1,2,3,4]) = 1.92 + var data1 = new double[] { 1, 2, 3, 4 }; + Assert.AreEqual(1.92, Numerics.Data.Statistics.Statistics.HarmonicMean(data1), 1E-10); + + // Data with zero should return NaN (not 0 or Infinity) + var data2 = new double[] { 1, 2, 0, 4 }; + Assert.AreEqual(double.NaN, Numerics.Data.Statistics.Statistics.HarmonicMean(data2)); + } } } diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeries.cs b/Test_Numerics/Data/Time Series/Test_TimeSeries.cs index e5989d64..8598f8e2 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeries.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeries.cs @@ -978,5 +978,92 @@ public void Test_PeaksOverThreshold() } + /// + /// Test the SeasonalDecompose method with a known sinusoidal + linear trend signal. + /// + [TestMethod] + public void Test_SeasonalDecompose() + { + // Create a time series: linear trend + sinusoidal seasonal + small noise + int period = 12; + int nYears = 5; + int n = period * nYears; // 60 data points + var ts = new TimeSeries(TimeInterval.OneMonth); + var startDate = new DateTime(2000, 1, 1); + + double[] truetrend = new double[n]; + double[] trueSeasonal = new double[n]; + + for (int i = 0; i < n; i++) + { + // Linear trend: 100 + 0.5 * i + truetrend[i] = 100.0 + 0.5 * i; + // Seasonal: 10 * sin(2π * i / 12) + trueSeasonal[i] = 10.0 * Math.Sin(2.0 * Math.PI * i / period); + double value = truetrend[i] + trueSeasonal[i]; + ts.Add(new SeriesOrdinate(startDate.AddMonths(i), value)); + } + + // Decompose + var (trend, seasonal, residual) = ts.SeasonalDecompose(period); + + // Verify trend is returned + Assert.IsGreaterThan(0, trend.Count, "Trend should have values."); + Assert.IsLessThanOrEqualTo(n, trend.Count, "Trend should not exceed original length."); + + // Verify seasonal has correct length + // HasCount does not accept double[] (not IEnumerable), so suppress MSTEST0037 +#pragma warning disable MSTEST0037 + Assert.AreEqual(n, seasonal.Length, "Seasonal should have same length as original."); +#pragma warning restore MSTEST0037 + + // Verify residual is returned + Assert.IsGreaterThan(0, residual.Count, "Residual should have values."); + + // Verify the decomposition is additive: original = trend + seasonal + residual + // This is an exact mathematical identity by construction + for (int i = 0; i < residual.Count; i++) + { + var residOrd = residual[i]; + var trendOrd = trend.FirstOrDefault(t => t.Index == residOrd.Index); + int origIdx = -1; + for (int k = 0; k < n; k++) + { + if (ts[k].Index == residOrd.Index) { origIdx = k; break; } + } + if (trendOrd != null && origIdx >= 0) + { + double reconstructed = trendOrd.Value + seasonal[origIdx] + residOrd.Value; + Assert.AreEqual(ts[origIdx].Value, reconstructed, 1E-6, + $"Additive decomposition should hold at index {origIdx}."); + } + } + + // The seasonal component should capture the sinusoidal signal + // Verify seasonal values are not all zero (they should have nonzero amplitude) + double maxSeasonal = seasonal.Max(); + double minSeasonal = seasonal.Min(); + Assert.IsGreaterThan(1.0, maxSeasonal - minSeasonal, + "Seasonal component should have nonzero amplitude for sinusoidal input."); + } + + /// + /// Test that SeasonalDecompose throws for invalid inputs. + /// + [TestMethod] + public void Test_SeasonalDecompose_InvalidInputs() + { + var ts = new TimeSeries(TimeInterval.OneMonth); + var startDate = new DateTime(2000, 1, 1); + for (int i = 0; i < 10; i++) + ts.Add(new SeriesOrdinate(startDate.AddMonths(i), i)); + + // Period too small + Assert.Throws(() => ts.SeasonalDecompose(1)); + + // Too few data points for 2 complete periods + Assert.Throws(() => ts.SeasonalDecompose(12)); + } + } } diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs index bf4444b8..19aa10e8 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs @@ -89,6 +89,16 @@ public class Test_TimeSeriesDownload /// private const string USGS_4 = "07010000"; + /// + /// USGS site number for Susquehanna River at Harrisburg, PA. + /// + private const string USGS_5 = "01570500"; + + /// + /// USGS site number for Potomac River at Little Falls near Washington, DC. + /// + private const string USGS_6 = "01646500"; + /// /// GHCN station: USC00040741. /// @@ -146,13 +156,13 @@ public class Test_TimeSeriesDownload private static void AssertDailySeriesMonotonic(TimeSeries ts) { Assert.IsNotNull(ts, "Time series is null."); - Assert.IsGreaterThan( 0, ts.Count); + Assert.IsGreaterThan(0, ts.Count); DateTime? prev = null; foreach (var pt in ts) { if (prev.HasValue) - Assert.IsTrue(pt.Index >= prev.Value, "Dates not sorted chronologically."); + Assert.IsGreaterThanOrEqualTo(prev.Value, pt.Index, "Dates not sorted chronologically."); prev = pt.Index; } @@ -280,14 +290,74 @@ await Assert.ThrowsAsync(async () => } /// - /// Ensures CHMN rejects unsupported time series types. + /// Ensures CHMN rejects unsupported time series types (field measurements not available). /// [TestMethod] public async Task CHMN_UnsupportedType_Throws() { await Assert.ThrowsAsync(async () => await TimeSeriesDownload.FromCHMN(CHMN_1, - TimeSeriesDownload.TimeSeriesType.PeakDischarge)); + TimeSeriesDownload.TimeSeriesType.MeasuredDischarge)); + } + + /// + /// Tests CHMN instantaneous discharge download (real-time 5-minute data). + /// + [TestMethod, TestCategory("Integration")] + public async Task CHMN_InstantaneousDischarge_Works() + { + if (!await Online()) return; + + var ts = await TimeSeriesDownload.FromCHMN(CHMN_2, + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); + } + + /// + /// Tests CHMN instantaneous stage download (real-time 5-minute data). + /// + [TestMethod, TestCategory("Integration")] + public async Task CHMN_InstantaneousStage_Works() + { + if (!await Online()) return; + + var ts = await TimeSeriesDownload.FromCHMN(CHMN_2, + TimeSeriesDownload.TimeSeriesType.InstantaneousStage); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); + } + + /// + /// Tests CHMN peak discharge download (annual instantaneous maximums). + /// + [TestMethod, TestCategory("Integration")] + public async Task CHMN_PeakDischarge_Works() + { + if (!await Online()) return; + + var ts = await TimeSeriesDownload.FromCHMN(CHMN_1, + TimeSeriesDownload.TimeSeriesType.PeakDischarge); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); + } + + /// + /// Tests CHMN peak stage download (annual instantaneous maximums). + /// + [TestMethod, TestCategory("Integration")] + public async Task CHMN_PeakStage_Works() + { + if (!await Online()) return; + + var ts = await TimeSeriesDownload.FromCHMN(CHMN_1, + TimeSeriesDownload.TimeSeriesType.PeakStage); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); } #endregion @@ -354,6 +424,98 @@ await TimeSeriesDownload.FromUSGS(USGS_1, TimeSeriesDownload.TimeSeriesType.DailyPrecipitation)); } + /// + /// Tests USGS field measurement discharge data retrieval from the OGC API. + /// + [TestMethod, TestCategory("Integration")] + public async Task USGS_MeasuredDischarge_Works() + { + if (!await Online()) return; + + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_5, + TimeSeriesDownload.TimeSeriesType.MeasuredDischarge); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsNotNull(raw, "Raw text is null."); + Assert.IsGreaterThan(0, ts.Count); + Assert.IsFalse(string.IsNullOrEmpty(raw), "Raw text should contain JSON response."); + + // Verify all values are positive (discharge must be > 0) + foreach (var pt in ts) + { + Assert.IsGreaterThan(0, pt.Value, $"Discharge value {pt.Value} at {pt.Index} should be positive."); + } + } + + /// + /// Tests USGS field measurement stage (gage height) data retrieval from the OGC API. + /// + [TestMethod, TestCategory("Integration")] + public async Task USGS_MeasuredStage_Works() + { + if (!await Online()) return; + + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_5, + TimeSeriesDownload.TimeSeriesType.MeasuredStage); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsNotNull(raw, "Raw text is null."); + Assert.IsGreaterThan(0, ts.Count); + Assert.IsFalse(string.IsNullOrEmpty(raw), "Raw text should contain JSON response."); + + // Verify all values are positive (gage height must be > 0) + foreach (var pt in ts) + { + Assert.IsGreaterThan(0, pt.Value, $"Gage height value {pt.Value} at {pt.Index} should be positive."); + } + } + + /// + /// Tests USGS peak stage data retrieval. + /// + [TestMethod, TestCategory("Integration")] + public async Task USGS_PeakStage_Works() + { + if (!await Online()) return; + + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_3, + TimeSeriesDownload.TimeSeriesType.PeakStage); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsNotNull(raw, "Raw text is null."); + Assert.IsGreaterThan(0, ts.Count); + } + + /// + /// Tests USGS instantaneous discharge download. + /// + [TestMethod, TestCategory("Integration")] + public async Task USGS_InstantaneousDischarge_Works() + { + if (!await Online()) return; + + var (ts, _) = await TimeSeriesDownload.FromUSGS(USGS_6, + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); + } + + /// + /// Tests USGS instantaneous stage download. + /// + [TestMethod, TestCategory("Integration")] + public async Task USGS_InstantaneousStage_Works() + { + if (!await Online()) return; + + var (ts, _) = await TimeSeriesDownload.FromUSGS(USGS_6, + TimeSeriesDownload.TimeSeriesType.InstantaneousStage); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); + } + #endregion #region GHCN Tests @@ -578,12 +740,81 @@ public async Task BOM_WindowedDownload_Works() var firstDate = ts.First().Index; var lastDate = ts.Last().Index; - Assert.IsTrue(firstDate >= WinStart.AddDays(-1), + Assert.IsGreaterThanOrEqualTo(WinStart.AddDays(-1), firstDate, $"First date {firstDate} is before window start {WinStart}"); - Assert.IsTrue(lastDate <= WinEnd.AddDays(1), + Assert.IsLessThanOrEqualTo(WinEnd.AddDays(1), lastDate, $"Last date {lastDate} is after window end {WinEnd}"); } + /// + /// Validates instantaneous discharge download from BOM. + /// + [TestMethod, TestCategory("Integration")] + public async Task BOM_InstantaneousDischarge_Works() + { + if (!await Online()) return; + var ts = await TimeSeriesDownload.FromABOM(BOM_1, + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge, + startDate: WinStart, endDate: WinEnd); + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); + } + + /// + /// Validates instantaneous stage download from BOM. + /// + [TestMethod, TestCategory("Integration")] + public async Task BOM_InstantaneousStage_Works() + { + if (!await Online()) return; + var ts = await TimeSeriesDownload.FromABOM(BOM_3, + TimeSeriesDownload.TimeSeriesType.InstantaneousStage, + startDate: WinStart, endDate: WinEnd); + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); + } + + /// + /// Validates daily precipitation download from BOM. + /// + [TestMethod, TestCategory("Integration")] + public async Task BOM_DailyPrecipitation_Works() + { + if (!await Online()) return; + var ts = await TimeSeriesDownload.FromABOM(BOM_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation, + startDate: WinStart, endDate: WinEnd); + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsGreaterThan(0, ts.Count); + } + + /// + /// Tests precipitation unit conversions (mm ↔ inches) for BOM data. + /// + [TestMethod, TestCategory("Integration")] + public async Task BOM_UnitConversion_Precip_MmIn() + { + if (!await Online()) return; + + var tsMm = await TimeSeriesDownload.FromABOM(BOM_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation, + depthUnit: TimeSeriesDownload.DepthUnit.Millimeters, + startDate: WinStart, endDate: WinEnd); + + var tsIn = await TimeSeriesDownload.FromABOM(BOM_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation, + depthUnit: TimeSeriesDownload.DepthUnit.Inches, + startDate: WinStart, endDate: WinEnd); + + const double factor = 25.4; + for (int i = 0; i < tsMm.Count; i++) + { + if (double.IsNaN(tsMm[i].Value) || double.IsNaN(tsIn[i].Value)) continue; + if (tsMm[i].Value == 0 && tsIn[i].Value == 0) continue; + AssertRoughlyEqual(tsMm[i].Value, tsIn[i].Value * factor); + } + } + #endregion } } diff --git a/Test_Numerics/Distributions/Bivariate Copulas/Test_AMHCopula.cs b/Test_Numerics/Distributions/Bivariate Copulas/Test_AMHCopula.cs index 6c8b9725..7845c52f 100644 --- a/Test_Numerics/Distributions/Bivariate Copulas/Test_AMHCopula.cs +++ b/Test_Numerics/Distributions/Bivariate Copulas/Test_AMHCopula.cs @@ -190,6 +190,22 @@ public void Test_MLE_Fit() Assert.AreEqual(83.76367, ((Normal)copula.MarginalDistributionY).Mu, 1E-2); Assert.AreEqual(29.76202, ((Normal)copula.MarginalDistributionY).Sigma, 1E-2); } + + /// + /// Test the tail dependence coefficients. + /// AMH copula: λ_U = λ_L = 0 (no tail dependence). + /// + [TestMethod] + public void Test_TailDependence() + { + var copula = new AMHCopula(0.5); + Assert.AreEqual(0.0, copula.UpperTailDependence, 1E-10, "AMH copula should have no upper tail dependence."); + Assert.AreEqual(0.0, copula.LowerTailDependence, 1E-10, "AMH copula should have no lower tail dependence."); + + var copulaNeg = new AMHCopula(-0.5); + Assert.AreEqual(0.0, copulaNeg.UpperTailDependence, 1E-10, "AMH copula should have no upper tail dependence for negative θ."); + Assert.AreEqual(0.0, copulaNeg.LowerTailDependence, 1E-10, "AMH copula should have no lower tail dependence for negative θ."); + } } } diff --git a/Test_Numerics/Distributions/Bivariate Copulas/Test_ClaytonCopula.cs b/Test_Numerics/Distributions/Bivariate Copulas/Test_ClaytonCopula.cs index 0347cfc0..0cbe3359 100644 --- a/Test_Numerics/Distributions/Bivariate Copulas/Test_ClaytonCopula.cs +++ b/Test_Numerics/Distributions/Bivariate Copulas/Test_ClaytonCopula.cs @@ -188,5 +188,30 @@ public void Test_MLE_Fit() Assert.AreEqual(80.54073, ((Normal)copula.MarginalDistributionY).Mu, 1E-2); Assert.AreEqual(26.11687, ((Normal)copula.MarginalDistributionY).Sigma, 1E-2); } + + /// + /// Test the tail dependence coefficients. + /// Clayton copula: λ_L = 2^(-1/θ), λ_U = 0. + /// + [TestMethod] + public void Test_TailDependence() + { + // θ = 2: λ_L = 2^(-1/2) ≈ 0.70711 + var copula = new ClaytonCopula(2.0); + Assert.AreEqual(0.0, copula.UpperTailDependence, 1E-10, "Clayton copula should have no upper tail dependence."); + Assert.AreEqual(Math.Pow(2.0, -0.5), copula.LowerTailDependence, 1E-6, "λ_L should equal 2^(-1/θ) for θ=2."); + + // θ = 1: λ_L = 2^(-1) = 0.5 + var copula1 = new ClaytonCopula(1.0); + Assert.AreEqual(0.5, copula1.LowerTailDependence, 1E-10, "λ_L should equal 0.5 for θ=1."); + + // θ → ∞: λ_L → 1 + var copulaLarge = new ClaytonCopula(100.0); + Assert.IsGreaterThan(0.99, copulaLarge.LowerTailDependence, "λ_L should approach 1 for large θ."); + + // θ ≤ 0: λ_L = 0 (no dependence or negative dependence) + var copulaZero = new ClaytonCopula(0.0); + Assert.AreEqual(0.0, copulaZero.LowerTailDependence, 1E-10, "λ_L should be 0 for θ ≤ 0."); + } } } diff --git a/Test_Numerics/Distributions/Bivariate Copulas/Test_FrankCopula.cs b/Test_Numerics/Distributions/Bivariate Copulas/Test_FrankCopula.cs index 5ef95788..cf76caab 100644 --- a/Test_Numerics/Distributions/Bivariate Copulas/Test_FrankCopula.cs +++ b/Test_Numerics/Distributions/Bivariate Copulas/Test_FrankCopula.cs @@ -175,5 +175,21 @@ public void Test_MLE_Fit() Assert.AreEqual(81.08905, ((Normal)copula.MarginalDistributionY).Mu, 1E-2); Assert.AreEqual(26.93537, ((Normal)copula.MarginalDistributionY).Sigma, 1E-2); } + + /// + /// Test the tail dependence coefficients. + /// Frank copula: λ_U = λ_L = 0 (no tail dependence). + /// + [TestMethod] + public void Test_TailDependence() + { + var copula = new FrankCopula(5.0); + Assert.AreEqual(0.0, copula.UpperTailDependence, 1E-10, "Frank copula should have no upper tail dependence."); + Assert.AreEqual(0.0, copula.LowerTailDependence, 1E-10, "Frank copula should have no lower tail dependence."); + + var copulaNeg = new FrankCopula(-5.0); + Assert.AreEqual(0.0, copulaNeg.UpperTailDependence, 1E-10, "Frank copula should have no upper tail dependence for negative θ."); + Assert.AreEqual(0.0, copulaNeg.LowerTailDependence, 1E-10, "Frank copula should have no lower tail dependence for negative θ."); + } } } diff --git a/Test_Numerics/Distributions/Bivariate Copulas/Test_GumbelCopula.cs b/Test_Numerics/Distributions/Bivariate Copulas/Test_GumbelCopula.cs index 15e082dc..04de9878 100644 --- a/Test_Numerics/Distributions/Bivariate Copulas/Test_GumbelCopula.cs +++ b/Test_Numerics/Distributions/Bivariate Copulas/Test_GumbelCopula.cs @@ -189,5 +189,30 @@ public void Test_MLE_Fit() Assert.AreEqual(24.83673, ((Normal)copula.MarginalDistributionY).Sigma, 1E-1); } + /// + /// Test the tail dependence coefficients. + /// Gumbel copula: λ_U = 2 - 2^(1/θ), λ_L = 0. + /// + [TestMethod] + public void Test_TailDependence() + { + // θ = 2: λ_U = 2 - 2^(1/2) ≈ 0.58579 + var copula = new GumbelCopula(2.0); + Assert.AreEqual(0.0, copula.LowerTailDependence, 1E-10, "Gumbel copula should have no lower tail dependence."); + Assert.AreEqual(2.0 - Math.Pow(2.0, 0.5), copula.UpperTailDependence, 1E-6, "λ_U should equal 2 - 2^(1/θ) for θ=2."); + + // θ = 4: λ_U = 2 - 2^(1/4) ≈ 0.81079 + var copula4 = new GumbelCopula(4.0); + Assert.AreEqual(2.0 - Math.Pow(2.0, 0.25), copula4.UpperTailDependence, 1E-6, "λ_U should equal 2 - 2^(1/θ) for θ=4."); + + // θ = 1 (independence): λ_U = 2 - 2 = 0 + var copulaIndep = new GumbelCopula(1.0); + Assert.AreEqual(0.0, copulaIndep.UpperTailDependence, 1E-10, "λ_U should be 0 for θ=1 (independence)."); + + // θ → ∞: λ_U → 1 + var copulaLarge = new GumbelCopula(100.0); + Assert.IsGreaterThan(0.99, copulaLarge.UpperTailDependence, "λ_U should approach 1 for large θ."); + } + } } diff --git a/Test_Numerics/Distributions/Bivariate Copulas/Test_JoeCopula.cs b/Test_Numerics/Distributions/Bivariate Copulas/Test_JoeCopula.cs index ad2926b5..991a4e66 100644 --- a/Test_Numerics/Distributions/Bivariate Copulas/Test_JoeCopula.cs +++ b/Test_Numerics/Distributions/Bivariate Copulas/Test_JoeCopula.cs @@ -178,5 +178,30 @@ public void Test_MLE_Fit() Assert.AreEqual(25.11105, ((Normal)copula.MarginalDistributionY).Sigma, 1E-2); } + /// + /// Test the tail dependence coefficients. + /// Joe copula: λ_U = 2 - 2^(1/θ), λ_L = 0. + /// + [TestMethod] + public void Test_TailDependence() + { + // θ = 2: λ_U = 2 - 2^(1/2) ≈ 0.58579 + var copula = new JoeCopula(2.0); + Assert.AreEqual(0.0, copula.LowerTailDependence, 1E-10, "Joe copula should have no lower tail dependence."); + Assert.AreEqual(2.0 - Math.Pow(2.0, 0.5), copula.UpperTailDependence, 1E-6, "λ_U should equal 2 - 2^(1/θ) for θ=2."); + + // θ = 4: λ_U = 2 - 2^(1/4) ≈ 0.81079 + var copula4 = new JoeCopula(4.0); + Assert.AreEqual(2.0 - Math.Pow(2.0, 0.25), copula4.UpperTailDependence, 1E-6, "λ_U should equal 2 - 2^(1/θ) for θ=4."); + + // θ = 1 (independence): λ_U = 2 - 2 = 0 + var copulaIndep = new JoeCopula(1.0); + Assert.AreEqual(0.0, copulaIndep.UpperTailDependence, 1E-10, "λ_U should be 0 for θ=1 (independence)."); + + // θ → ∞: λ_U → 1 + var copulaLarge = new JoeCopula(100.0); + Assert.IsGreaterThan(0.99, copulaLarge.UpperTailDependence, "λ_U should approach 1 for large θ."); + } + } } diff --git a/Test_Numerics/Distributions/Bivariate Copulas/Test_NormalCopula.cs b/Test_Numerics/Distributions/Bivariate Copulas/Test_NormalCopula.cs index 4f70d727..72938dd3 100644 --- a/Test_Numerics/Distributions/Bivariate Copulas/Test_NormalCopula.cs +++ b/Test_Numerics/Distributions/Bivariate Copulas/Test_NormalCopula.cs @@ -176,5 +176,40 @@ public void Test_MLE_Fit() Assert.AreEqual(84.17168, ((Normal)copula.MarginalDistributionY).Mu, 1E-2); Assert.AreEqual(24.58027, ((Normal)copula.MarginalDistributionY).Sigma, 1E-2); } + + /// + /// Verify NormalCopula CDF with u+v != 1 (where buggy and correct formulas differ). + /// Reference: scipy.stats.multivariate_normal.cdf([norm.ppf(u), norm.ppf(v)], cov=[[1,rho],[rho,1]]) + /// + [TestMethod] + public void Test_CDF_Asymmetric() + { + var copula = new NormalCopula(0.5); + Assert.AreEqual(0.6871505667, copula.CDF(0.8, 0.8), 1E-2); + + copula = new NormalCopula(-0.3); + Assert.AreEqual(0.1718571868, copula.CDF(0.3, 0.7), 1E-2); + Assert.AreEqual(0.2015066580, copula.CDF(0.5, 0.5), 1E-2); + + copula = new NormalCopula(0.9); + Assert.AreEqual(0.8688649404, copula.CDF(0.9, 0.9), 1E-2); + } + + /// + /// Test the tail dependence coefficients. + /// Normal copula: λ_U = λ_L = 0 (no tail dependence). + /// + [TestMethod] + public void Test_TailDependence() + { + var copula = new NormalCopula(0.5); + Assert.AreEqual(0.0, copula.UpperTailDependence, 1E-10, "Normal copula should have no upper tail dependence."); + Assert.AreEqual(0.0, copula.LowerTailDependence, 1E-10, "Normal copula should have no lower tail dependence."); + + // Even with high correlation, Normal copula has zero tail dependence + var copulaHigh = new NormalCopula(0.99); + Assert.AreEqual(0.0, copulaHigh.UpperTailDependence, 1E-10, "Normal copula should have no tail dependence even with high ρ."); + Assert.AreEqual(0.0, copulaHigh.LowerTailDependence, 1E-10, "Normal copula should have no tail dependence even with high ρ."); + } } } diff --git a/Test_Numerics/Distributions/Bivariate Copulas/Test_StudentTCopula.cs b/Test_Numerics/Distributions/Bivariate Copulas/Test_StudentTCopula.cs new file mode 100644 index 00000000..41d3f350 --- /dev/null +++ b/Test_Numerics/Distributions/Bivariate Copulas/Test_StudentTCopula.cs @@ -0,0 +1,537 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Data.Statistics; +using Numerics.Distributions; +using Numerics.Distributions.Copulas; +using Numerics.Sampling.MCMC; + +namespace Distributions.BivariateCopulas +{ + /// + /// Unit tests for the Student's t-Copula. + /// + /// + /// + /// Authors: + /// + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + /// References: + /// + /// + /// Reference values verified against R 'copula' package (tCopula, dCopula, pCopula). + /// + /// + [TestClass] + public class Test_StudentTCopula + { + + /// + /// Test default construction. + /// + [TestMethod] + public void Test_Construction() + { + var copula = new StudentTCopula(); + Assert.AreEqual(0.0, copula.Theta); + Assert.AreEqual(5, copula.DegreesOfFreedom); + Assert.AreEqual(CopulaType.StudentT, copula.Type); + Assert.AreEqual("Student's t", copula.DisplayName); + Assert.AreEqual("t", copula.ShortDisplayName); + Assert.IsTrue(copula.ParametersValid); + } + + /// + /// Test parameterized construction. + /// + [TestMethod] + public void Test_ParameterizedConstruction() + { + var copula = new StudentTCopula(0.5, 10); + Assert.AreEqual(0.5, copula.Theta); + Assert.AreEqual(10, copula.DegreesOfFreedom); + Assert.IsTrue(copula.ParametersValid); + } + + /// + /// Test invalid degrees of freedom. + /// + [TestMethod] + public void Test_InvalidDegreesOfFreedom() + { + Assert.Throws(() => new StudentTCopula(0.5, 2)); + Assert.Throws(() => new StudentTCopula(0.5, 0)); + Assert.Throws(() => new StudentTCopula(0.5, -1)); + } + + /// + /// Test invalid correlation parameter. + /// + [TestMethod] + public void Test_InvalidTheta() + { + var copula = new StudentTCopula(-2.0, 5); + Assert.IsFalse(copula.ParametersValid); + + copula = new StudentTCopula(2.0, 5); + Assert.IsFalse(copula.ParametersValid); + } + + /// + /// Test ParameterToString property. + /// + [TestMethod] + public void Test_ParameterToString() + { + var copula = new StudentTCopula(0.5, 10); + var parms = copula.ParameterToString; + Assert.AreEqual("Correlation (ρ)", parms[0, 0]); + Assert.AreEqual("0.5", parms[0, 1]); + Assert.AreEqual("Degrees of Freedom (ν)", parms[1, 0]); + Assert.AreEqual("10", parms[1, 1]); + } + + /// + /// Test the PDF of the t-copula. + /// + /// + /// The t-copula density approaches the Normal copula density as ν → ∞. + /// For finite ν, the t-copula density differs from the Normal copula density, + /// particularly in the tails. + /// + [TestMethod] + public void Test_PDF() + { + // t-copula with ν=5, ρ=0.5 at (0.3, 0.7) + // The copula density should be well-defined and positive + var copula = new StudentTCopula(0.5, 5); + double pdf = copula.PDF(0.3, 0.7); + Assert.IsGreaterThan(0, pdf, "PDF should be positive."); + + // At the center (0.5, 0.5), the density should be relatively high + double pdfCenter = copula.PDF(0.5, 0.5); + Assert.IsGreaterThan(0, pdfCenter); + + // Symmetry: c(u,v; ρ) = c(v,u; ρ) for elliptical copulas + Assert.AreEqual(copula.PDF(0.2, 0.8), copula.PDF(0.8, 0.2), 1E-6); + Assert.AreEqual(copula.PDF(0.3, 0.7), copula.PDF(0.7, 0.3), 1E-6); + Assert.AreEqual(copula.PDF(0.1, 0.9), copula.PDF(0.9, 0.1), 1E-6); + + // With ρ = 0, the copula density should be close to (but not exactly) 1 at the center + var copulaIndep = new StudentTCopula(0.0, 100); + double pdfIndep = copulaIndep.PDF(0.5, 0.5); + // For large ν and ρ=0, should approach the independence copula density (= 1) + Assert.AreEqual(1.0, pdfIndep, 0.1); + } + + /// + /// Test t-copula PDF consistency: the copula density should integrate to 1 over [0,1]^2 + /// (approximately verified by trapezoidal quadrature). + /// + [TestMethod] + public void Test_PDF_Integration() + { + var copula = new StudentTCopula(0.5, 5); + + // Numerical integration via trapezoidal rule + int n = 50; + double h = 1.0 / n; + double sum = 0; + for (int i = 1; i < n; i++) + { + for (int j = 1; j < n; j++) + { + double u = i * h; + double v = j * h; + sum += copula.PDF(u, v); + } + } + double integral = sum * h * h; + Assert.AreEqual(1.0, integral, 0.05); // Should be close to 1 + } + + /// + /// Test the CDF of the t-copula. + /// + [TestMethod] + public void Test_CDF() + { + // Basic boundary checks + var copula = new StudentTCopula(0.5, 5); + + // CDF should be symmetric for elliptical copulas + Assert.AreEqual(copula.CDF(0.2, 0.8), copula.CDF(0.8, 0.2), 1E-2); + + // CDF should be between 0 and 1 + double cdf = copula.CDF(0.3, 0.7); + Assert.IsGreaterThanOrEqualTo(0.0, cdf); + Assert.IsLessThanOrEqualTo(1.0, cdf); + + // CDF(u, 1) ≈ u for all copulas (Fréchet-Hoeffding bound) + Assert.AreEqual(0.3, copula.CDF(0.3, 0.99999), 1E-2); + + // CDF(1, v) ≈ v for all copulas + Assert.AreEqual(0.7, copula.CDF(0.99999, 0.7), 1E-2); + + // High positive correlation should yield CDF ≈ min(u,v) + var copulaHigh = new StudentTCopula(0.99, 5); + Assert.AreEqual(0.2, copulaHigh.CDF(0.2, 0.8), 0.05); + + // Independent case (ρ=0, high ν): CDF ≈ u*v + var copulaIndep = new StudentTCopula(0.0, 100); + Assert.AreEqual(0.2 * 0.8, copulaIndep.CDF(0.2, 0.8), 0.02); + } + + /// + /// Test that the CDF is monotonically non-decreasing in both arguments. + /// + [TestMethod] + public void Test_CDF_Monotonicity() + { + var copula = new StudentTCopula(0.5, 5); + double v = 0.5; + double prevCdf = 0; + for (double u = 0.05; u <= 0.95; u += 0.1) + { + double cdf = copula.CDF(u, v); + Assert.IsGreaterThanOrEqualTo(prevCdf - 1E-10, cdf, $"CDF not monotone at u={u}"); + prevCdf = cdf; + } + } + + /// + /// Test InverseCDF round-trip: CDF(InverseCDF(u, v)) ≈ (u, v). + /// + [TestMethod] + public void Test_InverseCDF_RoundTrip() + { + var copula = new StudentTCopula(0.6, 5); + var rng = new Random(42); + + for (int i = 0; i < 20; i++) + { + double u = 0.05 + 0.9 * rng.NextDouble(); + double v = 0.05 + 0.9 * rng.NextDouble(); + + var result = copula.InverseCDF(u, v); + Assert.AreEqual(u, result[0], 1E-10, "First component should be unchanged."); + Assert.IsGreaterThanOrEqualTo(0, result[1], "Second component should be in [0,1]."); + Assert.IsLessThanOrEqualTo(1, result[1], "Second component should be in [0,1]."); + } + } + + /// + /// Test that InverseCDF samples produce the correct dependence structure. + /// + [TestMethod] + public void Test_InverseCDF_Dependence() + { + // With high positive ρ, InverseCDF(u, v) should produce correlated pairs + var copula = new StudentTCopula(0.8, 5); + int n = 1000; + var rng = new Random(42); + double sumProduct = 0; + double sumU = 0, sumV = 0; + + for (int i = 0; i < n; i++) + { + double u = rng.NextDouble(); + double v = rng.NextDouble(); + var result = copula.InverseCDF(u, v); + sumU += result[0]; + sumV += result[1]; + sumProduct += result[0] * result[1]; + } + + double meanU = sumU / n; + double meanV = sumV / n; + double cov = sumProduct / n - meanU * meanV; + + // Covariance should be positive for positive ρ + Assert.IsGreaterThan(0, cov, "Samples should show positive dependence."); + } + + /// + /// Test the tail dependence coefficients. + /// + /// + /// The t-copula has symmetric tail dependence: + /// λ_U = λ_L = 2·t_{ν+1}(-√((ν+1)(1-ρ)/(1+ρ))) + /// + [TestMethod] + public void Test_TailDependence() + { + // For ρ = 0.5, ν = 4: + // λ = 2·t_5(-√(5·0.5/1.5)) = 2·t_5(-√(5/3)) = 2·t_5(-1.2910) + var copula = new StudentTCopula(0.5, 4); + double lambdaU = copula.UpperTailDependence; + double lambdaL = copula.LowerTailDependence; + Assert.IsGreaterThan(0, lambdaU, "Upper tail dependence should be positive for finite ν."); + Assert.IsLessThan(1, lambdaU, "Upper tail dependence should be less than 1."); + Assert.AreEqual(lambdaU, lambdaL, 1E-10, "t-copula tail dependence should be symmetric."); + + // ρ = 0, ν = 4: still has tail dependence (this is the key difference from Normal copula) + var copulaZero = new StudentTCopula(0.0, 4); + double lambdaZero = copulaZero.UpperTailDependence; + Assert.IsGreaterThan(0, lambdaZero, "t-copula with ρ=0 still has tail dependence."); + + // Higher ν → lower tail dependence (approaching Normal copula with λ=0) + var copulaHighNu = new StudentTCopula(0.5, 100); + double lambdaHighNu = copulaHighNu.UpperTailDependence; + Assert.IsLessThan(lambdaU, lambdaHighNu, "Higher ν should reduce tail dependence."); + + // Higher ρ → higher tail dependence + var copulaHighRho = new StudentTCopula(0.9, 4); + Assert.IsGreaterThan(lambdaU, copulaHighRho.UpperTailDependence, "Higher ρ should increase tail dependence."); + + // ρ close to -1: tail dependence approaches 0 + var copulaNegRho = new StudentTCopula(-0.99, 4); + Assert.IsLessThan(0.01, copulaNegRho.UpperTailDependence, "Tail dependence should be near 0 for ρ ≈ -1."); + } + + /// + /// Test that the t-copula converges to the Normal copula as ν → ∞. + /// + [TestMethod] + public void Test_ConvergenceToNormal() + { + double rho = 0.5; + var normalCopula = new NormalCopula(rho); + var tCopulaHighNu = new StudentTCopula(rho, 1000); + + // PDF should converge + double normalPdf = normalCopula.PDF(0.3, 0.7); + double tPdf = tCopulaHighNu.PDF(0.3, 0.7); + Assert.AreEqual(normalPdf, tPdf, 0.05); + + // CDF should converge + double normalCdf = normalCopula.CDF(0.3, 0.7); + double tCdf = tCopulaHighNu.CDF(0.3, 0.7); + Assert.AreEqual(normalCdf, tCdf, 0.05); + } + + /// + /// Test random sampling generates valid copula samples. + /// + [TestMethod] + public void Test_Sampling() + { + var copula = new StudentTCopula(0.6, 5); + var samples = copula.GenerateRandomValues(500, seed: 42); + + for (int i = 0; i < 500; i++) + { + // All values should be in [0, 1] + Assert.IsGreaterThanOrEqualTo(0, samples[i, 0], $"Sample [{i},0] out of range."); + Assert.IsLessThanOrEqualTo(1, samples[i, 0], $"Sample [{i},0] out of range."); + Assert.IsGreaterThanOrEqualTo(0, samples[i, 1], $"Sample [{i},1] out of range."); + Assert.IsLessThanOrEqualTo(1, samples[i, 1], $"Sample [{i},1] out of range."); + } + } + + /// + /// Test Clone produces an independent copy. + /// + [TestMethod] + public void Test_Clone() + { + var copula = new StudentTCopula(0.5, 10); + var clone = copula.Clone() as StudentTCopula; + Assert.IsNotNull(clone); + Assert.AreEqual(copula.Theta, clone.Theta); + Assert.AreEqual(copula.DegreesOfFreedom, clone.DegreesOfFreedom); + Assert.IsTrue(clone.ParametersValid); + + // Mutating clone should not affect original + clone.Theta = 0.1; + Assert.AreEqual(0.5, copula.Theta); + } + + private double[] data1 = new double[] { 122.094066003419, 92.8321267206161, 86.4920318705377, 87.6183663113541, 102.558777787492, 103.627475117762, 127.084948716539, 105.908684131013, 110.065795957654, 105.924647125867, 110.009738155469, 126.490833800772, 64.1264871206211, 81.3150800229481, 92.0780134395721, 106.040322550555, 113.158086143066, 117.051057784044, 127.110531266645, 108.907371862136, 105.476247114194, 108.629403495407, 98.7803988364997, 93.217925588845, 97.7219451830075, 109.178093756809, 137.69504856252, 106.884615327674, 112.139177456202, 85.7416217661797, 71.0610938629716, 112.644166631765, 119.545871678548, 70.5169833274982, 99.6896817997206, 100.987892854545, 103.659280253554, 75.6075621013066, 118.810868919796, 109.113664695226, 113.636425353944, 100.008375355612, 113.178917359795, 80.4269472604342, 88.3638384448237, 90.2905074656314, 98.7995143316863, 98.4698060067802, 108.279297570816, 86.1578437055905, 101.183725242941, 85.5531148952956, 111.024195253862, 121.934506174556, 104.169993666179, 84.4652994609478, 99.6099259747033, 95.3130792386208, 115.45680252817, 120.213139478586, 95.5691788140058, 92.7950300448044, 102.58430893827, 86.7105161576407, 82.8059368562185, 107.335705516294, 112.603259240932, 102.780778760832, 128.958090528336, 105.139162595628, 118.272661482198, 99.8275937885748, 94.2856024560543, 108.48679008009, 100.147734981682, 88.7006383425785, 89.6441478272035, 112.24266306884, 99.8184811468069, 120.592090049738, 124.023170133661, 101.250961381805, 90.0000027551006, 108.781064635426, 94.9203320035987, 99.9491821782837, 88.7473944659517, 94.3643253649856, 105.814317118952, 92.6866900633813, 111.020330544613, 111.676189456988, 115.70235103978, 124.659106152655, 81.3866270495082, 120.178528245778, 93.6511977805724, 114.099368762143, 119.062045395294, 74.1998497412903 }; + private double[] data2 = new double[] { 127.869024514059, 53.5970265830273, 35.6871183968043, 77.5937820397885, 84.619117510857, 110.477376636164, 114.679535976765, 109.338354392258, 88.5987759167264, 72.6695216679034, 111.932652280673, 86.3677960278751, 23.9336347978345, 51.2377830227977, 82.4565771813309, 92.9162733515069, 117.465381827514, 104.862362549521, 131.059266136887, 67.2743851584176, 100.263235166171, 113.734275000025, 73.1582387829997, 78.4353197703676, 60.0180359279642, 106.709991071405, 123.175455301514, 98.7006449949188, 99.860486991242, 55.7603096813567, 53.7716423706874, 104.659445447656, 119.899401349887, 59.8670226375024, 94.0117104763717, 101.424610891155, 114.256354904191, 53.5051841563538, 118.35993465227, 73.1605008375787, 87.4677698350712, 75.4031529479113, 105.404958657365, 53.336411944238, 61.2731445424292, 72.377272009744, 88.959659863884, 80.1301183393358, 98.624093971352, 81.9603074727622, 52.0788199186743, 75.49358652998, 90.2428259997917, 101.326931349259, 48.1343463500222, 56.9295918059918, 89.0348875829931, 69.0012535890253, 100.355241744174, 74.00820280539, 63.9482913881998, 64.4973782209222, 95.8934144135508, 85.4028102356618, 37.8958459664423, 99.2194777630975, 126.581868541047, 91.8287794302242, 143.543939198862, 108.751405708845, 100.951567564812, 73.5051068155712, 83.419507788205, 84.9090133796832, 59.3886126411711, 84.0348703304947, 78.1503115396303, 104.953483626903, 77.6450718557069, 117.615613165515, 118.131904013699, 76.3190144944821, 62.0183469143453, 97.4729901076061, 49.3396925267253, 58.6790714873228, 45.0596168059506, 85.3857426310419, 65.0772008397323, 58.8836242438228, 79.2838406333912, 102.608529398935, 83.7509120512927, 103.106132785215, 52.8403092456187, 88.4802383528401, 64.2906982187616, 93.0489784548541, 116.065815369284, 26.2779209375887 }; + + /// + /// Test fitting with the method of maximum pseudo likelihood. + /// + [TestMethod] + public void Test_MPL_Fit() + { + // get ranks of data + var rank1 = Statistics.RanksInPlace(data1); + var rank2 = Statistics.RanksInPlace(data2); + // get plotting positions + for (int i = 0; i < data1.Length; i++) + { + rank1[i] = rank1[i] / (rank1.Length + 1d); + rank2[i] = rank2[i] / (rank2.Length + 1d); + } + // Fit copula (fix ν=5, estimate ρ) + BivariateCopula copula = new StudentTCopula(0.0, 5); + BivariateCopulaEstimation.Estimate(ref copula, rank1, rank2, CopulaEstimationMethod.PseudoLikelihood); + + // The estimated ρ should be positive and reasonable for this correlated data + Assert.IsGreaterThan(0.5, copula.Theta, $"Estimated ρ = {copula.Theta} should be > 0.5"); + Assert.IsLessThan(1.0, copula.Theta, $"Estimated ρ = {copula.Theta} should be < 1.0"); + } + + /// + /// Estimate using the inference from margins method. + /// + [TestMethod] + public void Test_IFM_Fit() + { + BivariateCopula copula = new StudentTCopula(0.0, 5); + copula.MarginalDistributionX = new Normal(); + copula.MarginalDistributionY = new Normal(); + // Fit marginals + ((IEstimation)copula.MarginalDistributionX).Estimate(data1, ParameterEstimationMethod.MaximumLikelihood); + ((IEstimation)copula.MarginalDistributionY).Estimate(data2, ParameterEstimationMethod.MaximumLikelihood); + // Fit copula + BivariateCopulaEstimation.Estimate(ref copula, data1, data2, CopulaEstimationMethod.InferenceFromMargins); + + // The estimated ρ should be positive and reasonable + Assert.IsGreaterThan(0.5, copula.Theta, $"Estimated ρ = {copula.Theta} should be > 0.5"); + Assert.IsLessThan(1.0, copula.Theta, $"Estimated ρ = {copula.Theta} should be < 1.0"); + } + + /// + /// Test that MPL estimates both rho and degrees of freedom. + /// + [TestMethod] + public void Test_MPL_EstimatesBothParameters() + { + var rank1 = Statistics.RanksInPlace(data1); + var rank2 = Statistics.RanksInPlace(data2); + for (int i = 0; i < data1.Length; i++) + { + rank1[i] = rank1[i] / (rank1.Length + 1d); + rank2[i] = rank2[i] / (rank2.Length + 1d); + } + + // Start with default ν=5, the estimation should update it + BivariateCopula copula = new StudentTCopula(0.0, 5); + BivariateCopulaEstimation.Estimate(ref copula, rank1, rank2, CopulaEstimationMethod.PseudoLikelihood); + + var tCopula = (StudentTCopula)copula; + + // ρ should be estimated as positive + Assert.IsGreaterThan(0.5, tCopula.Theta, $"Estimated ρ = {tCopula.Theta} should be > 0.5"); + Assert.IsLessThan(1.0, tCopula.Theta, $"Estimated ρ = {tCopula.Theta} should be < 1.0"); + + // ν should be estimated (may differ from initial value of 5) + Assert.IsGreaterThanOrEqualTo(3, tCopula.DegreesOfFreedom, $"Estimated ν = {tCopula.DegreesOfFreedom} should be >= 3"); + Assert.IsLessThanOrEqualTo(60, tCopula.DegreesOfFreedom, $"Estimated ν = {tCopula.DegreesOfFreedom} should be <= 60"); + + // Tail dependence should be data-driven (not user-defined) + double lambda = tCopula.UpperTailDependence; + Assert.IsGreaterThan(0, lambda, "Tail dependence should be positive."); + Assert.IsLessThan(1, lambda, "Tail dependence should be less than 1."); + } + + /// + /// Test that IFM estimates both rho and degrees of freedom. + /// + [TestMethod] + public void Test_IFM_EstimatesBothParameters() + { + BivariateCopula copula = new StudentTCopula(0.0, 5); + copula.MarginalDistributionX = new Normal(); + copula.MarginalDistributionY = new Normal(); + ((IEstimation)copula.MarginalDistributionX).Estimate(data1, ParameterEstimationMethod.MaximumLikelihood); + ((IEstimation)copula.MarginalDistributionY).Estimate(data2, ParameterEstimationMethod.MaximumLikelihood); + + BivariateCopulaEstimation.Estimate(ref copula, data1, data2, CopulaEstimationMethod.InferenceFromMargins); + + var tCopula = (StudentTCopula)copula; + Assert.IsGreaterThan(0.5, tCopula.Theta, $"Estimated ρ = {tCopula.Theta} should be > 0.5"); + Assert.IsGreaterThanOrEqualTo(3, tCopula.DegreesOfFreedom, $"Estimated ν = {tCopula.DegreesOfFreedom} should be >= 3"); + Assert.IsLessThanOrEqualTo(60, tCopula.DegreesOfFreedom, $"Estimated ν = {tCopula.DegreesOfFreedom} should be <= 60"); + } + + /// + /// Test GetCopulaParameters and SetCopulaParameters round-trip. + /// + [TestMethod] + public void Test_GetSetCopulaParameters() + { + var copula = new StudentTCopula(0.7, 10); + + // GetCopulaParameters should return [rho, nu] + Assert.AreEqual(2, copula.NumberOfCopulaParameters); + var parms = copula.GetCopulaParameters; + Assert.AreEqual(0.7, parms[0], 1E-10); + Assert.AreEqual(10.0, parms[1], 1E-10); + + // SetCopulaParameters should update both + copula.SetCopulaParameters(new double[] { -0.3, 15.0 }); + Assert.AreEqual(-0.3, copula.Theta, 1E-10); + Assert.AreEqual(15, copula.DegreesOfFreedom); + + // SetCopulaParameters should round nu to nearest integer + copula.SetCopulaParameters(new double[] { 0.5, 7.6 }); + Assert.AreEqual(8, copula.DegreesOfFreedom); + + copula.SetCopulaParameters(new double[] { 0.5, 7.4 }); + Assert.AreEqual(7, copula.DegreesOfFreedom); + + // SetCopulaParameters should clamp nu to minimum of 3 + copula.SetCopulaParameters(new double[] { 0.5, 1.0 }); + Assert.AreEqual(3, copula.DegreesOfFreedom); + } + + /// + /// Test ParameterConstraints returns correct 2D array. + /// + [TestMethod] + public void Test_ParameterConstraints() + { + var copula = new StudentTCopula(0.5, 5); + var constraints = copula.ParameterConstraints(data1, data2); + + // Should be [2, 2] array + Assert.AreEqual(2, constraints.GetLength(0)); + Assert.AreEqual(2, constraints.GetLength(1)); + + // Row 0: rho constraints [-1+eps, 1-eps] + Assert.IsLessThan(-0.99, constraints[0, 0]); + Assert.IsGreaterThan(0.99, constraints[0, 1]); + + // Row 1: nu constraints [3, 60] + Assert.AreEqual(3.0, constraints[1, 0]); + Assert.AreEqual(60.0, constraints[1, 1]); + } + + } +} diff --git a/Test_Numerics/Distributions/Multivariate/Test_Dirichlet.cs b/Test_Numerics/Distributions/Multivariate/Test_Dirichlet.cs new file mode 100644 index 00000000..7da69d0e --- /dev/null +++ b/Test_Numerics/Distributions/Multivariate/Test_Dirichlet.cs @@ -0,0 +1,328 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Distributions; + +namespace Distributions.Multivariate +{ + /// + /// Unit tests for the Dirichlet distribution. + /// + /// + /// + /// Authors: + /// + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + /// References: + /// + /// + /// Reference values verified analytically and against R package MCMCpack::ddirichlet. + /// + /// + [TestClass] + public class Test_Dirichlet + { + + /// + /// Test symmetric Dirichlet construction. + /// + [TestMethod] + public void Test_SymmetricConstruction() + { + var D = new Dirichlet(3, 2.0); + Assert.AreEqual(3, D.Dimension); + Assert.IsTrue(D.ParametersValid); + + var alpha = D.Alpha; + Assert.AreEqual(2.0, alpha[0]); + Assert.AreEqual(2.0, alpha[1]); + Assert.AreEqual(2.0, alpha[2]); + Assert.AreEqual(6.0, D.AlphaSum); + } + + /// + /// Test asymmetric Dirichlet construction. + /// + [TestMethod] + public void Test_AsymmetricConstruction() + { + var D = new Dirichlet(new double[] { 1.0, 2.0, 3.0 }); + Assert.AreEqual(3, D.Dimension); + Assert.IsTrue(D.ParametersValid); + Assert.AreEqual(6.0, D.AlphaSum); + } + + /// + /// Test invalid parameters. + /// + [TestMethod] + public void Test_InvalidParameters() + { + Assert.Throws(() => new Dirichlet(1, 1.0)); // dimension < 2 + Assert.Throws(() => new Dirichlet(3, 0.0)); // alpha = 0 + Assert.Throws(() => new Dirichlet(3, -1.0)); // alpha < 0 + Assert.Throws(() => new Dirichlet(new double[] { 1.0, -1.0 })); // negative alpha + } + + /// + /// Test distribution type and display names. + /// + [TestMethod] + public void Test_TypeAndName() + { + var D = new Dirichlet(3, 1.0); + Assert.AreEqual(MultivariateDistributionType.Dirichlet, D.Type); + Assert.AreEqual("Dirichlet", D.DisplayName); + Assert.AreEqual("Dir", D.ShortDisplayName); + } + + /// + /// Test mean vector. Mean[i] = alpha[i] / sum(alpha). + /// + [TestMethod] + public void Test_Mean() + { + // Symmetric Dir(2, 2, 2): mean = (1/3, 1/3, 1/3) + var D1 = new Dirichlet(3, 2.0); + var mean1 = D1.Mean; + Assert.AreEqual(1.0 / 3.0, mean1[0], 1e-10); + Assert.AreEqual(1.0 / 3.0, mean1[1], 1e-10); + Assert.AreEqual(1.0 / 3.0, mean1[2], 1e-10); + + // Asymmetric Dir(1, 2, 3): mean = (1/6, 2/6, 3/6) + var D2 = new Dirichlet(new double[] { 1.0, 2.0, 3.0 }); + var mean2 = D2.Mean; + Assert.AreEqual(1.0 / 6.0, mean2[0], 1e-10); + Assert.AreEqual(2.0 / 6.0, mean2[1], 1e-10); + Assert.AreEqual(3.0 / 6.0, mean2[2], 1e-10); + } + + /// + /// Test variance vector. Var[i] = alpha[i] * (S - alpha[i]) / (S^2 * (S+1)). + /// + [TestMethod] + public void Test_Variance() + { + // Dir(1, 2, 3): S = 6 + // Var[0] = 1*5 / (36*7) = 5/252 ≈ 0.019841 + // Var[1] = 2*4 / (36*7) = 8/252 ≈ 0.031746 + // Var[2] = 3*3 / (36*7) = 9/252 ≈ 0.035714 + var D = new Dirichlet(new double[] { 1.0, 2.0, 3.0 }); + var v = D.Variance; + Assert.AreEqual(5.0 / 252.0, v[0], 1e-10); + Assert.AreEqual(8.0 / 252.0, v[1], 1e-10); + Assert.AreEqual(9.0 / 252.0, v[2], 1e-10); + } + + /// + /// Test mode vector. Mode[i] = (alpha[i] - 1) / (S - K) when all alpha > 1. + /// + [TestMethod] + public void Test_Mode() + { + // Dir(2, 3, 5): S=10, K=3, S-K=7 + // Mode = (1/7, 2/7, 4/7) + var D = new Dirichlet(new double[] { 2.0, 3.0, 5.0 }); + var mode = D.Mode; + Assert.AreEqual(1.0 / 7.0, mode[0], 1e-10); + Assert.AreEqual(2.0 / 7.0, mode[1], 1e-10); + Assert.AreEqual(4.0 / 7.0, mode[2], 1e-10); + } + + /// + /// Test that Mode throws when any alpha <= 1. + /// + [TestMethod] + public void Test_Mode_Invalid() + { + var D = new Dirichlet(new double[] { 0.5, 2.0, 3.0 }); + Assert.Throws(() => { var m = D.Mode; }); + } + + /// + /// Test covariance. Cov(Xi, Xj) = -alpha_i * alpha_j / (S^2 * (S+1)). + /// + [TestMethod] + public void Test_Covariance() + { + var D = new Dirichlet(new double[] { 1.0, 2.0, 3.0 }); + // Cov(X0, X1) = -1*2 / (36*7) = -2/252 + Assert.AreEqual(-2.0 / 252.0, D.Covariance(0, 1), 1e-10); + // Cov(X0, X0) = Var(X0) + Assert.AreEqual(D.Variance[0], D.Covariance(0, 0), 1e-10); + // Covariance matrix is symmetric + Assert.AreEqual(D.Covariance(0, 1), D.Covariance(1, 0), 1e-10); + } + + /// + /// Test the PDF for the symmetric Dir(1,1,1) = Uniform on simplex. + /// + [TestMethod] + public void Test_PDF_Uniform() + { + // Dir(1,1,1) is uniform on the 2-simplex. + // PDF = Gamma(3) / (Gamma(1)*Gamma(1)*Gamma(1)) = 2! = 2 + var D = new Dirichlet(3, 1.0); + double pdf = D.PDF(new double[] { 1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0 }); + Assert.AreEqual(2.0, pdf, 1e-6); + + // Should be the same everywhere on the simplex + double pdf2 = D.PDF(new double[] { 0.1, 0.2, 0.7 }); + Assert.AreEqual(2.0, pdf2, 1e-6); + } + + /// + /// Test the PDF for Dir(2, 3, 5). + /// + [TestMethod] + public void Test_PDF_Asymmetric() + { + // Dir(2,3,5): B(alpha) = Gamma(2)*Gamma(3)*Gamma(5) / Gamma(10) + // = 1! * 2! * 4! / 9! = 1*2*24 / 362880 = 48/362880 = 1/7560 + // PDF(0.1, 0.3, 0.6) = (1/B) * 0.1^1 * 0.3^2 * 0.6^4 + // = 7560 * 0.1 * 0.09 * 0.1296 = 7560 * 0.0011664 = 8.81... + var D = new Dirichlet(new double[] { 2.0, 3.0, 5.0 }); + double pdf = D.PDF(new double[] { 0.1, 0.3, 0.6 }); + double expected = 7560.0 * 0.1 * 0.09 * 0.1296; + Assert.AreEqual(expected, pdf, 1e-4); + } + + /// + /// Test that PDF returns 0 for points outside the simplex. + /// + [TestMethod] + public void Test_PDF_OutsideSimplex() + { + var D = new Dirichlet(3, 2.0); + // Components don't sum to 1 + Assert.AreEqual(0.0, D.PDF(new double[] { 0.5, 0.5, 0.5 }), 1e-10); + // Negative component + Assert.AreEqual(0.0, D.PDF(new double[] { -0.1, 0.6, 0.5 }), 1e-10); + // Wrong dimension + Assert.AreEqual(0.0, D.PDF(new double[] { 0.5, 0.5 }), 1e-10); + } + + /// + /// Test that LogPDF is consistent with PDF. + /// + [TestMethod] + public void Test_LogPDF() + { + var D = new Dirichlet(new double[] { 2.0, 3.0, 5.0 }); + var x = new double[] { 0.2, 0.3, 0.5 }; + Assert.AreEqual(Math.Log(D.PDF(x)), D.LogPDF(x), 1e-10); + } + + /// + /// Test random sampling: all samples on the simplex and moments converge. + /// + [TestMethod] + public void Test_Sampling() + { + var D = new Dirichlet(new double[] { 2.0, 3.0, 5.0 }); + var samples = D.GenerateRandomValues(10000, seed: 42); + + // All samples should be on the simplex + for (int i = 0; i < 10000; i++) + { + double sum = 0; + for (int j = 0; j < 3; j++) + { + Assert.IsGreaterThan(0, samples[i, j], $"Sample [{i},{j}] = {samples[i, j]} is not positive"); + sum += samples[i, j]; + } + Assert.AreEqual(1.0, sum, 1e-10, $"Sample {i} does not sum to 1"); + } + + // Check that sample means converge to theoretical means + var sampleMeans = new double[3]; + for (int i = 0; i < 10000; i++) + { + for (int j = 0; j < 3; j++) + sampleMeans[j] += samples[i, j]; + } + for (int j = 0; j < 3; j++) + sampleMeans[j] /= 10000; + + var theoreticalMeans = D.Mean; + Assert.AreEqual(theoreticalMeans[0], sampleMeans[0], 0.02); // 2/10 = 0.2 + Assert.AreEqual(theoreticalMeans[1], sampleMeans[1], 0.02); // 3/10 = 0.3 + Assert.AreEqual(theoreticalMeans[2], sampleMeans[2], 0.02); // 5/10 = 0.5 + } + + /// + /// Test Clone produces an independent copy. + /// + [TestMethod] + public void Test_Clone() + { + var D = new Dirichlet(new double[] { 1.0, 2.0, 3.0 }); + var D2 = D.Clone() as Dirichlet; + Assert.IsNotNull(D2); + Assert.AreEqual(D.Dimension, D2.Dimension); + Assert.IsTrue(D2.ParametersValid); + + // Verify alpha values are copied + var a1 = D.Alpha; + var a2 = D2.Alpha; + for (int i = 0; i < D.Dimension; i++) + Assert.AreEqual(a1[i], a2[i]); + } + + /// + /// Test the LogMultivariateBeta function. + /// + [TestMethod] + public void Test_LogMultivariateBeta() + { + // For alpha = (1, 1), B(1,1) = Gamma(1)*Gamma(1)/Gamma(2) = 1*1/1 = 1 + Assert.AreEqual(0.0, Dirichlet.LogMultivariateBeta(new double[] { 1.0, 1.0 }), 1e-10); + + // For alpha = (1, 1, 1), B(1,1,1) = Gamma(1)^3/Gamma(3) = 1/2 + Assert.AreEqual(Math.Log(0.5), Dirichlet.LogMultivariateBeta(new double[] { 1.0, 1.0, 1.0 }), 1e-10); + } + + /// + /// Test CDF throws NotImplementedException. + /// + [TestMethod] + public void Test_CDF_Throws() + { + var D = new Dirichlet(3, 1.0); + Assert.Throws(() => D.CDF(new double[] { 0.3, 0.3, 0.4 })); + } + + } +} diff --git a/Test_Numerics/Distributions/Multivariate/Test_Multinomial.cs b/Test_Numerics/Distributions/Multivariate/Test_Multinomial.cs new file mode 100644 index 00000000..858c48c3 --- /dev/null +++ b/Test_Numerics/Distributions/Multivariate/Test_Multinomial.cs @@ -0,0 +1,287 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Distributions; + +namespace Distributions.Multivariate +{ + /// + /// Unit tests for the Multinomial distribution. + /// + /// + /// + /// Authors: + /// + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + /// References: + /// + /// + /// Reference values verified analytically and against R dmultinom. + /// + /// + [TestClass] + public class Test_Multinomial + { + + /// + /// Test basic construction. + /// + [TestMethod] + public void Test_Construction() + { + var M = new Multinomial(10, new double[] { 0.2, 0.3, 0.5 }); + Assert.AreEqual(3, M.Dimension); + Assert.AreEqual(10, M.NumberOfTrials); + Assert.IsTrue(M.ParametersValid); + } + + /// + /// Test invalid parameters. + /// + [TestMethod] + public void Test_InvalidParameters() + { + Assert.Throws(() => new Multinomial(0, new double[] { 0.5, 0.5 })); + Assert.Throws(() => new Multinomial(10, new double[] { 0.5 })); // too few categories + Assert.Throws(() => new Multinomial(10, new double[] { -0.1, 0.6, 0.5 })); // negative prob + Assert.Throws(() => new Multinomial(10, new double[] { 0.3, 0.3, 0.3 })); // don't sum to 1 + } + + /// + /// Test type and display names. + /// + [TestMethod] + public void Test_TypeAndName() + { + var M = new Multinomial(10, new double[] { 0.5, 0.5 }); + Assert.AreEqual(MultivariateDistributionType.Multinomial, M.Type); + Assert.AreEqual("Multinomial", M.DisplayName); + Assert.AreEqual("Mult", M.ShortDisplayName); + } + + /// + /// Test mean vector. Mean[i] = N * p[i]. + /// + [TestMethod] + public void Test_Mean() + { + var M = new Multinomial(10, new double[] { 0.2, 0.3, 0.5 }); + var mean = M.Mean; + Assert.AreEqual(2.0, mean[0], 1e-10); + Assert.AreEqual(3.0, mean[1], 1e-10); + Assert.AreEqual(5.0, mean[2], 1e-10); + } + + /// + /// Test variance vector. Var[i] = N * p[i] * (1 - p[i]). + /// + [TestMethod] + public void Test_Variance() + { + var M = new Multinomial(10, new double[] { 0.2, 0.3, 0.5 }); + var variance = M.Variance; + Assert.AreEqual(10 * 0.2 * 0.8, variance[0], 1e-10); + Assert.AreEqual(10 * 0.3 * 0.7, variance[1], 1e-10); + Assert.AreEqual(10 * 0.5 * 0.5, variance[2], 1e-10); + } + + /// + /// Test covariance. Cov(Xi, Xj) = -N * pi * pj. + /// + [TestMethod] + public void Test_Covariance() + { + var M = new Multinomial(10, new double[] { 0.2, 0.3, 0.5 }); + Assert.AreEqual(-10 * 0.2 * 0.3, M.Covariance(0, 1), 1e-10); + Assert.AreEqual(M.Variance[0], M.Covariance(0, 0), 1e-10); + } + + /// + /// Test PMF for a simple fair coin case: Mult(10, (0.5, 0.5)) = Binomial(10, 0.5). + /// + [TestMethod] + public void Test_PMF_Binomial() + { + // Mult(10, (0.5, 0.5)) at x = (5, 5) should equal C(10,5) * 0.5^10 + // = 252 * 0.0009765625 = 0.24609375 + var M = new Multinomial(10, new double[] { 0.5, 0.5 }); + double pmf = M.PDF(new double[] { 5, 5 }); + Assert.AreEqual(0.24609375, pmf, 1e-8); + } + + /// + /// Test PMF for a 3-category case. + /// + [TestMethod] + public void Test_PMF_ThreeCategory() + { + // Mult(4, (0.2, 0.3, 0.5)) at x = (1, 1, 2) + // = 4! / (1! 1! 2!) * 0.2^1 * 0.3^1 * 0.5^2 + // = 24 / (1*1*2) * 0.2 * 0.3 * 0.25 + // = 12 * 0.015 = 0.18 + var M = new Multinomial(4, new double[] { 0.2, 0.3, 0.5 }); + double pmf = M.PDF(new double[] { 1, 1, 2 }); + Assert.AreEqual(0.18, pmf, 1e-10); + } + + /// + /// Test that PMF returns 0 for invalid count vectors. + /// + [TestMethod] + public void Test_PMF_Invalid() + { + var M = new Multinomial(10, new double[] { 0.5, 0.5 }); + // Counts don't sum to N + Assert.AreEqual(0.0, M.PDF(new double[] { 3, 3 }), 1e-10); + // Negative count + Assert.AreEqual(0.0, M.PDF(new double[] { -1, 11 }), 1e-10); + // Wrong dimension + Assert.AreEqual(0.0, M.PDF(new double[] { 5, 3, 2 }), 1e-10); + } + + /// + /// Test LogPMF consistency with PMF. + /// + [TestMethod] + public void Test_LogPMF() + { + var M = new Multinomial(10, new double[] { 0.2, 0.3, 0.5 }); + var x = new double[] { 2, 3, 5 }; + Assert.AreEqual(Math.Log(M.PDF(x)), M.LogPMF(x), 1e-10); + } + + /// + /// Test random sampling: each sample sums to N and moments converge. + /// + [TestMethod] + public void Test_Sampling() + { + var M = new Multinomial(100, new double[] { 0.2, 0.3, 0.5 }); + var samples = M.GenerateRandomValues(5000, seed: 42); + + // All samples should sum to N + for (int i = 0; i < 5000; i++) + { + double sum = 0; + for (int j = 0; j < 3; j++) + { + Assert.IsGreaterThanOrEqualTo(0, samples[i, j], $"Sample [{i},{j}] is negative"); + sum += samples[i, j]; + } + Assert.AreEqual(100.0, sum, 1e-10, $"Sample {i} does not sum to N"); + } + + // Sample means should converge to theoretical means + var sampleMeans = new double[3]; + for (int i = 0; i < 5000; i++) + for (int j = 0; j < 3; j++) + sampleMeans[j] += samples[i, j]; + + var theoreticalMeans = M.Mean; + for (int j = 0; j < 3; j++) + { + sampleMeans[j] /= 5000; + Assert.AreEqual(theoreticalMeans[j], sampleMeans[j], 1.0); // within 1 count of N*p + } + } + + /// + /// Test the static Sample method for weighted categorical sampling. + /// + [TestMethod] + public void Test_WeightedSample() + { + var rng = new Random(42); + var weights = new double[] { 1.0, 3.0, 6.0 }; // 10%, 30%, 60% + var counts = new int[3]; + + int n = 10000; + for (int i = 0; i < n; i++) + { + int idx = Multinomial.Sample(weights, rng); + Assert.IsGreaterThanOrEqualTo(0, idx); + Assert.IsLessThan(3, idx); + counts[idx]++; + } + + // Check proportions + Assert.AreEqual(0.1, counts[0] / (double)n, 0.02); + Assert.AreEqual(0.3, counts[1] / (double)n, 0.02); + Assert.AreEqual(0.6, counts[2] / (double)n, 0.02); + } + + /// + /// Test the static Sample method with edge cases. + /// + [TestMethod] + public void Test_WeightedSample_EdgeCases() + { + var rng = new Random(42); + // All weight on one category + Assert.AreEqual(1, Multinomial.Sample(new double[] { 0.0, 1.0, 0.0 }, rng)); + + // Single positive weight + Assert.AreEqual(0, Multinomial.Sample(new double[] { 5.0, 0.0 }, rng)); + + // Invalid: all zeros + Assert.Throws(() => Multinomial.Sample(new double[] { 0.0, 0.0 }, rng)); + } + + /// + /// Test Clone produces an independent copy. + /// + [TestMethod] + public void Test_Clone() + { + var M = new Multinomial(10, new double[] { 0.2, 0.3, 0.5 }); + var M2 = M.Clone() as Multinomial; + Assert.IsNotNull(M2); + Assert.AreEqual(M.Dimension, M2.Dimension); + Assert.AreEqual(M.NumberOfTrials, M2.NumberOfTrials); + Assert.IsTrue(M2.ParametersValid); + } + + /// + /// Test CDF throws NotImplementedException. + /// + [TestMethod] + public void Test_CDF_Throws() + { + var M = new Multinomial(10, new double[] { 0.5, 0.5 }); + Assert.Throws(() => M.CDF(new double[] { 5, 5 })); + } + + } +} diff --git a/Test_Numerics/Distributions/Multivariate/Test_MultivariateNormal.cs b/Test_Numerics/Distributions/Multivariate/Test_MultivariateNormal.cs index de4cced4..8f177c10 100644 --- a/Test_Numerics/Distributions/Multivariate/Test_MultivariateNormal.cs +++ b/Test_Numerics/Distributions/Multivariate/Test_MultivariateNormal.cs @@ -78,16 +78,16 @@ public void Test_MultivariateNormalDist() double cdf = MultiN.CDF(new[] { 3d, 5d }); for (int i = 0; i <= MultiN.Dimension - 1; i++) { - Assert.AreEqual(mean[i], true_mean[i], 0.0001d); - Assert.AreEqual(median[i], true_median[i], 0.0001d); - Assert.AreEqual(mode[i], true_mode[i], 0.0001d); - Assert.AreEqual(stdev[i], true_stdDev[i], 0.0001d); + Assert.AreEqual(true_mean[i], mean[i], 0.0001d); + Assert.AreEqual(true_median[i], median[i], 0.0001d); + Assert.AreEqual(true_mode[i], mode[i], 0.0001d); + Assert.AreEqual(true_stdDev[i], stdev[i], 0.0001d); } - Assert.AreEqual(pdf1, true_pdf1, 0.0001d); - Assert.AreEqual(pdf2, true_pdf2, 0.0001d); - Assert.AreEqual(pdf3, true_pdf3, 0.0001d); - Assert.AreEqual(cdf, true_cdf, 0.0001d); + Assert.AreEqual(true_pdf1, pdf1, 0.0001d); + Assert.AreEqual(true_pdf2, pdf2, 0.0001d); + Assert.AreEqual(true_pdf3, pdf3, 0.0001d); + Assert.AreEqual(true_cdf, cdf, 0.0001d); } @@ -145,11 +145,11 @@ public void Test_MultivariateNormalCDF_R() // AB var p = mvn.CDF(new[] { Normal.StandardZ(0.25), Normal.StandardZ(0.35), double.PositiveInfinity, double.PositiveInfinity }); - Assert.AreEqual(0.05011069, p, 1E-4); + Assert.AreEqual(0.05011069, p, 1E-4); // AC p = mvn.CDF(new[] { Normal.StandardZ(0.25), double.PositiveInfinity, Normal.StandardZ(0.5), double.PositiveInfinity }); - Assert.AreEqual(0.0827451, p, 1E-4); + Assert.AreEqual(0.0827451, p, 1E-4); // AD p = mvn.CDF(new[] { Normal.StandardZ(0.25), double.PositiveInfinity, double.PositiveInfinity, Normal.StandardZ(0.5) }); diff --git a/Test_Numerics/Distributions/Multivariate/Test_MultivariateStudentT.cs b/Test_Numerics/Distributions/Multivariate/Test_MultivariateStudentT.cs new file mode 100644 index 00000000..230e5281 --- /dev/null +++ b/Test_Numerics/Distributions/Multivariate/Test_MultivariateStudentT.cs @@ -0,0 +1,970 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Distributions; +using Numerics.Sampling; + +namespace Distributions.Multivariate +{ + /// + /// Unit tests for the Multivariate Student's t-distribution. + /// + /// + /// + /// Authors: + /// + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + /// Reference values verified against Python scipy.stats.multivariate_t (v1.11+). + /// + /// + [TestClass] + public class Test_MultivariateStudentT + { + + #region Test Parameters + + /// + /// Standard test case: 2D MVT with ν=5, μ=(1,2), Σ=((1,0.5),(0.5,2)). + /// + private static MultivariateStudentT CreateStandard2D() + { + return new MultivariateStudentT( + 5.0, + new[] { 1.0, 2.0 }, + new[,] { { 1.0, 0.5 }, { 0.5, 2.0 } }); + } + + #endregion + + #region Properties Tests + + /// + /// Verify basic distribution properties: Mean, Mode, Median, Variance, StandardDeviation, Covariance. + /// Verified against analytical formulas. + /// + [TestMethod] + public void Test_Properties() + { + var mvt = CreateStandard2D(); + + // Mean = location for ν > 1 + Assert.AreEqual(1.0, mvt.Mean[0], 1E-10); + Assert.AreEqual(2.0, mvt.Mean[1], 1E-10); + + // Mode = location + Assert.AreEqual(1.0, mvt.Mode[0], 1E-10); + Assert.AreEqual(2.0, mvt.Mode[1], 1E-10); + + // Median = location + Assert.AreEqual(1.0, mvt.Median[0], 1E-10); + Assert.AreEqual(2.0, mvt.Median[1], 1E-10); + + // Variance = ν/(ν-2) · diag(Σ) = 5/3 · [1, 2] = [1.6667, 3.3333] + Assert.AreEqual(1.6666666666666667, mvt.Variance[0], 1E-10); + Assert.AreEqual(3.3333333333333335, mvt.Variance[1], 1E-10); + + // StandardDeviation + Assert.AreEqual(Math.Sqrt(5.0 / 3.0), mvt.StandardDeviation[0], 1E-10); + Assert.AreEqual(Math.Sqrt(10.0 / 3.0), mvt.StandardDeviation[1], 1E-10); + + // Covariance = ν/(ν-2) · Σ + var cov = mvt.Covariance; + Assert.AreEqual(1.6666666666666667, cov[0, 0], 1E-10); + Assert.AreEqual(0.8333333333333333, cov[0, 1], 1E-10); + Assert.AreEqual(0.8333333333333333, cov[1, 0], 1E-10); + Assert.AreEqual(3.3333333333333335, cov[1, 1], 1E-10); + + // Other properties + Assert.AreEqual(5.0, mvt.DegreesOfFreedom, 1E-10); + Assert.AreEqual(2, mvt.Dimension); + Assert.IsTrue(mvt.ParametersValid); + Assert.IsTrue(mvt.IsPositiveDefinite); + Assert.AreEqual("Multivariate Student's t", mvt.DisplayName); + Assert.AreEqual("Multi-T", mvt.ShortDisplayName); + Assert.AreEqual(MultivariateDistributionType.MultivariateStudentT, mvt.Type); + } + + /// + /// Verify that Mean throws for ν ≤ 1 and Variance throws for ν ≤ 2. + /// + [TestMethod] + public void Test_UndefinedMoments() + { + // ν = 1: Mean undefined + var mvt1 = new MultivariateStudentT(2, 1.0); + Assert.Throws(() => { var _ = mvt1.Mean; }); + + // ν = 2: Variance undefined + var mvt2 = new MultivariateStudentT(2, 2.0); + Assert.Throws(() => { var _ = mvt2.Variance; }); + Assert.Throws(() => { var _ = mvt2.Covariance; }); + + // ν = 1: Variance also undefined + Assert.Throws(() => { var _ = mvt1.Variance; }); + + // ν = 3: Both defined + var mvt3 = new MultivariateStudentT(2, 3.0); + Assert.IsNotNull(mvt3.Mean); + Assert.IsNotNull(mvt3.Variance); + } + + #endregion + + #region PDF Tests + + /// + /// Verify PDF against Python scipy.stats.multivariate_t for 2D case. + /// MVT(ν=5, μ=(1,2), Σ=((1,0.5),(0.5,2))). + /// + [TestMethod] + public void Test_PDF() + { + var mvt = CreateStandard2D(); + + // At mode (x = μ) + double pdf_mode = mvt.PDF(new[] { 1.0, 2.0 }); + Assert.AreEqual(0.12030982838508346, pdf_mode, 1E-12); + + // Away from mode + double pdf_00 = mvt.PDF(new[] { 0.0, 0.0 }); + Assert.AreEqual(0.032213925218971075, pdf_00, 1E-12); + + double pdf_34 = mvt.PDF(new[] { 3.0, 4.0 }); + Assert.AreEqual(0.012395882561151605, pdf_34, 1E-12); + + double pdf_25 = mvt.PDF(new[] { 2.0, 5.0 }); + Assert.AreEqual(0.012395882561151605, pdf_25, 1E-12); + + double pdf_37 = mvt.PDF(new[] { 3.0, 7.0 }); + Assert.AreEqual(0.0013219841225890843, pdf_37, 1E-12); + } + + /// + /// Verify LogPDF is consistent with log(PDF). + /// + [TestMethod] + public void Test_LogPDF() + { + var mvt = CreateStandard2D(); + + var testPoints = new[] + { + new[] { 1.0, 2.0 }, + new[] { 0.0, 0.0 }, + new[] { 3.0, 4.0 }, + new[] { -1.0, 5.0 } + }; + + foreach (var x in testPoints) + { + double pdf = mvt.PDF(x); + double logpdf = mvt.LogPDF(x); + Assert.AreEqual(Math.Log(pdf), logpdf, 1E-10, + $"LogPDF inconsistent with log(PDF) at ({x[0]}, {x[1]})"); + } + + // Known values + Assert.AreEqual(-2.1176849603770576, mvt.LogPDF(new[] { 1.0, 2.0 }), 1E-10); + Assert.AreEqual(-3.4353564596992499, mvt.LogPDF(new[] { 0.0, 0.0 }), 1E-10); + } + + /// + /// Verify that 1D MVT PDF matches univariate Student's t PDF. + /// MVT(ν=5, μ=[3], Σ=[[4]]) should equal StudentT(μ=3, σ=2, ν=5). + /// + [TestMethod] + public void Test_PDF_ReducesToUnivariate() + { + double nu = 5.0; + double mu = 3.0; + double sigma = 2.0; + + // Note: The Numerics StudentT.PDF returns the standard t density evaluated at + // Z = (x-μ)/σ, without the 1/σ Jacobian factor. So: + // StudentT.PDF(x) = σ · MVT(ν, [μ], [[σ²]]).PDF(x) + var mvt = new MultivariateStudentT(nu, new[] { mu }, new[,] { { sigma * sigma } }); + var univT = new StudentT(mu, sigma, nu); + + double[] testX = { -5.0, -1.0, 0.0, 3.0, 5.0, 10.0 }; + foreach (double x in testX) + { + double mvtPdf = mvt.PDF(new[] { x }); + double univPdf = univT.PDF(x); + Assert.AreEqual(univPdf / sigma, mvtPdf, 1E-12, + $"1D MVT PDF does not match univariate StudentT PDF at x={x}"); + } + } + + #endregion + + #region Mahalanobis Tests + + /// + /// Verify Mahalanobis distance computation against analytical values. + /// For Σ=((1,0.5),(0.5,2)): Σ⁻¹ = (1/1.75)·((2,-0.5),(-0.5,1)). + /// + [TestMethod] + public void Test_Mahalanobis() + { + var mvt = CreateStandard2D(); + + // At mode: Q = 0 + Assert.AreEqual(0.0, mvt.Mahalanobis(new[] { 1.0, 2.0 }), 1E-12); + + // At (0, 0): z = (-1, -2), Q = z'Σ⁻¹z + // Σ⁻¹ = (1/1.75)·((2,-0.5),(-0.5,1)) + // Q = (1/1.75)[(-1)²·2 + 2·(-1)·(-2)·(-0.5) + (-2)²·1] + // = (1/1.75)[2 - 2 + 4] = 4/1.75 ≈ 2.2857 + Assert.AreEqual(2.2857142857142856, mvt.Mahalanobis(new[] { 0.0, 0.0 }), 1E-10); + + // At (3, 4): z = (2, 2), Q = (1/1.75)[4·2 + 2·2·2·(-0.5) + 4·1] + // = (1/1.75)[8 - 4 + 4] = 8/1.75 ≈ 4.5714 + Assert.AreEqual(4.5714285714285712, mvt.Mahalanobis(new[] { 3.0, 4.0 }), 1E-10); + } + + /// + /// Verify that Mahalanobis throws for wrong dimension. + /// + [TestMethod] + public void Test_Mahalanobis_WrongDimension() + { + var mvt = CreateStandard2D(); + Assert.Throws(() => + mvt.Mahalanobis(new[] { 1.0, 2.0, 3.0 })); + } + + #endregion + + #region CDF Tests + + /// + /// Verify 1D CDF matches univariate Student's t CDF. + /// + [TestMethod] + public void Test_CDF_1D() + { + double nu = 5.0; + double mu = 3.0; + double sigma = 2.0; + + var mvt = new MultivariateStudentT(nu, new[] { mu }, new[,] { { sigma * sigma } }); + var univT = new StudentT(mu, sigma, nu); + + // CDF at x=5 + Assert.AreEqual(univT.CDF(5.0), mvt.CDF(new[] { 5.0 }), 1E-10); + // CDF at x=3 (median) + Assert.AreEqual(0.5, mvt.CDF(new[] { 3.0 }), 1E-10); + // CDF at x=0 + Assert.AreEqual(univT.CDF(0.0), mvt.CDF(new[] { 0.0 }), 1E-10); + } + + /// + /// Verify 2D CDF against Monte Carlo reference values from scipy. + /// These are approximate (MC tolerance ≈ 0.01). + /// + [TestMethod] + public void Test_CDF_2D() + { + var mvt = CreateStandard2D(); + + // CDF at (1, 2) ≈ 0.3076 (MC, N=10M) + double cdf_12 = mvt.CDF(new[] { 1.0, 2.0 }); + Assert.AreEqual(0.3076, cdf_12, 0.01, + $"CDF at (1,2) = {cdf_12:F4}, expected ≈ 0.3076"); + + // CDF at (3, 5) ≈ 0.9166 (MC, N=10M) + double cdf_35 = mvt.CDF(new[] { 3.0, 5.0 }); + Assert.AreEqual(0.9166, cdf_35, 0.01, + $"CDF at (3,5) = {cdf_35:F4}, expected ≈ 0.9166"); + + // CDF at (0, 0) ≈ 0.0454 (MC, N=10M) + double cdf_00 = mvt.CDF(new[] { 0.0, 0.0 }); + Assert.AreEqual(0.0454, cdf_00, 0.01, + $"CDF at (0,0) = {cdf_00:F4}, expected ≈ 0.0454"); + } + + #endregion + + #region Sampling Tests + + /// + /// Verify that GenerateRandomValues produces samples with correct mean and covariance. + /// For ν=5: E[X] = μ, Cov[X] = ν/(ν-2)·Σ = (5/3)·Σ. + /// + [TestMethod] + public void Test_Sampling_MeanCovariance() + { + var mvt = CreateStandard2D(); + int N = 100000; + var samples = mvt.GenerateRandomValues(N, 12345); + + // Compute sample mean + double mean0 = 0, mean1 = 0; + for (int i = 0; i < N; i++) + { + mean0 += samples[i, 0]; + mean1 += samples[i, 1]; + } + mean0 /= N; + mean1 /= N; + + Assert.AreEqual(1.0, mean0, 0.05, $"Sample mean[0] = {mean0:F4}"); + Assert.AreEqual(2.0, mean1, 0.05, $"Sample mean[1] = {mean1:F4}"); + + // Compute sample covariance + double cov00 = 0, cov01 = 0, cov11 = 0; + for (int i = 0; i < N; i++) + { + double d0 = samples[i, 0] - mean0; + double d1 = samples[i, 1] - mean1; + cov00 += d0 * d0; + cov01 += d0 * d1; + cov11 += d1 * d1; + } + cov00 /= (N - 1); + cov01 /= (N - 1); + cov11 /= (N - 1); + + // Theoretical: ν/(ν-2) · Σ = (5/3) · [[1, 0.5], [0.5, 2]] + Assert.AreEqual(5.0 / 3.0, cov00, 0.1, $"Sample Cov[0,0] = {cov00:F4}"); + Assert.AreEqual(5.0 / 6.0, cov01, 0.1, $"Sample Cov[0,1] = {cov01:F4}"); + Assert.AreEqual(10.0 / 3.0, cov11, 0.2, $"Sample Cov[1,1] = {cov11:F4}"); + } + + /// + /// Verify that LatinHypercubeRandomValues produces samples with correct mean and covariance. + /// + [TestMethod] + public void Test_LHS_MeanCovariance() + { + var mvt = CreateStandard2D(); + int N = 100000; + var samples = mvt.LatinHypercubeRandomValues(N, 12345); + + // Compute sample mean + double mean0 = 0, mean1 = 0; + for (int i = 0; i < N; i++) + { + mean0 += samples[i, 0]; + mean1 += samples[i, 1]; + } + mean0 /= N; + mean1 /= N; + + Assert.AreEqual(1.0, mean0, 0.05, $"LHS mean[0] = {mean0:F4}"); + Assert.AreEqual(2.0, mean1, 0.05, $"LHS mean[1] = {mean1:F4}"); + + // Compute sample covariance + double cov00 = 0, cov01 = 0, cov11 = 0; + for (int i = 0; i < N; i++) + { + double d0 = samples[i, 0] - mean0; + double d1 = samples[i, 1] - mean1; + cov00 += d0 * d0; + cov01 += d0 * d1; + cov11 += d1 * d1; + } + cov00 /= (N - 1); + cov01 /= (N - 1); + cov11 /= (N - 1); + + Assert.AreEqual(5.0 / 3.0, cov00, 0.1, $"LHS Cov[0,0] = {cov00:F4}"); + Assert.AreEqual(5.0 / 6.0, cov01, 0.1, $"LHS Cov[0,1] = {cov01:F4}"); + Assert.AreEqual(10.0 / 3.0, cov11, 0.2, $"LHS Cov[1,1] = {cov11:F4}"); + } + + /// + /// Verify that StratifiedRandomValues produces valid samples without errors. + /// + [TestMethod] + public void Test_StratifiedRandomValues() + { + var mvt = CreateStandard2D(); + + // Create stratification bins from uniform quantiles + var bins = new System.Collections.Generic.List(); + int nBins = 100; + for (int i = 0; i < nBins; i++) + { + double lower = (double)i / nBins; + double upper = (double)(i + 1) / nBins; + bins.Add(new StratificationBin(lower, upper, 1)); + } + + var samples = mvt.StratifiedRandomValues(bins, 12345); + + // Basic checks: correct dimensions, finite values + Assert.AreEqual(nBins, samples.GetLength(0)); + Assert.AreEqual(2, samples.GetLength(1)); + + for (int i = 0; i < nBins; i++) + { + Assert.IsFalse(double.IsNaN(samples[i, 0]), $"NaN at sample [{i}, 0]"); + Assert.IsFalse(double.IsNaN(samples[i, 1]), $"NaN at sample [{i}, 1]"); + Assert.IsFalse(double.IsInfinity(samples[i, 0]), $"Inf at sample [{i}, 0]"); + Assert.IsFalse(double.IsInfinity(samples[i, 1]), $"Inf at sample [{i}, 1]"); + } + } + + /// + /// Verify that marginal distributions of samples follow univariate Student's t. + /// Each marginal of MVT(ν, μ, Σ) is StudentT(μ_i, √Σ_ii, ν). + /// Test via Kolmogorov-Smirnov style check on empirical CDF. + /// + [TestMethod] + public void Test_Sampling_MarginalDistribution() + { + var mvt = CreateStandard2D(); + int N = 50000; + var samples = mvt.GenerateRandomValues(N, 54321); + + // Extract first marginal and sort + var x0 = new double[N]; + for (int i = 0; i < N; i++) + x0[i] = samples[i, 0]; + Array.Sort(x0); + + // Theoretical marginal: StudentT(μ=1, σ=√1=1, ν=5) + var marginal = new StudentT(1.0, 1.0, 5.0); + + // Compute max |empirical CDF - theoretical CDF| + double maxDiff = 0; + for (int i = 0; i < N; i++) + { + double empiricalCdf = (i + 1.0) / N; + double theoreticalCdf = marginal.CDF(x0[i]); + maxDiff = Math.Max(maxDiff, Math.Abs(empiricalCdf - theoreticalCdf)); + } + + // KS critical value for N=50000 at α=0.01 ≈ 1.63/√N ≈ 0.0073 + Assert.IsLessThan(0.01, maxDiff, + $"KS statistic = {maxDiff:F5}, marginal does not match StudentT(1, 1, 5)"); + } + + #endregion + + #region Convergence Tests + + /// + /// Verify that with very large ν, MVT PDF converges to MVN PDF. + /// + [TestMethod] + public void Test_HighDF_ConvergesToNormal_PDF() + { + double nu = 100000.0; + var location = new[] { 1.0, 2.0 }; + var sigma = new[,] { { 1.0, 0.5 }, { 0.5, 2.0 } }; + + var mvt = new MultivariateStudentT(nu, location, sigma); + var mvn = new MultivariateNormal(location, sigma); + + var testPoints = new[] + { + new[] { 1.0, 2.0 }, + new[] { 0.0, 0.0 }, + new[] { 3.0, 4.0 }, + new[] { -1.0, 5.0 } + }; + + foreach (var x in testPoints) + { + double pdfT = mvt.PDF(x); + double pdfN = mvn.PDF(x); + double relErr = Math.Abs(pdfT - pdfN) / pdfN; + Assert.IsLessThan(5E-4, relErr, + $"PDF mismatch at ({x[0]},{x[1]}): MVT={pdfT:E6}, MVN={pdfN:E6}, relErr={relErr:E3}"); + } + } + + /// + /// Verify that with very large ν, MVT samples converge to MVN samples (same covariance). + /// For large ν, Cov → Σ (since ν/(ν-2) → 1). + /// + [TestMethod] + public void Test_HighDF_ConvergesToNormal_Sampling() + { + double nu = 100000.0; + var location = new[] { 1.0, 2.0 }; + var sigma = new[,] { { 1.0, 0.5 }, { 0.5, 2.0 } }; + + var mvt = new MultivariateStudentT(nu, location, sigma); + int N = 50000; + var samples = mvt.GenerateRandomValues(N, 12345); + + // Compute sample covariance + double mean0 = 0, mean1 = 0; + for (int i = 0; i < N; i++) + { + mean0 += samples[i, 0]; + mean1 += samples[i, 1]; + } + mean0 /= N; + mean1 /= N; + + double cov00 = 0, cov01 = 0, cov11 = 0; + for (int i = 0; i < N; i++) + { + double d0 = samples[i, 0] - mean0; + double d1 = samples[i, 1] - mean1; + cov00 += d0 * d0; + cov01 += d0 * d1; + cov11 += d1 * d1; + } + cov00 /= (N - 1); + cov01 /= (N - 1); + cov11 /= (N - 1); + + // For large ν, Cov → Σ + Assert.AreEqual(1.0, cov00, 0.05, $"Cov[0,0] = {cov00:F4}"); + Assert.AreEqual(0.5, cov01, 0.05, $"Cov[0,1] = {cov01:F4}"); + Assert.AreEqual(2.0, cov11, 0.1, $"Cov[1,1] = {cov11:F4}"); + } + + #endregion + + #region Validation Tests + + /// + /// Verify that invalid parameters are properly rejected. + /// + [TestMethod] + public void Test_ParameterValidation_NegativeDF() + { + Assert.Throws(() => + new MultivariateStudentT(-1.0, new[] { 0.0 }, new[,] { { 1.0 } })); + } + + /// + /// Verify that zero degrees of freedom is rejected. + /// + [TestMethod] + public void Test_ParameterValidation_ZeroDF() + { + Assert.Throws(() => + new MultivariateStudentT(0.0, new[] { 0.0 }, new[,] { { 1.0 } })); + } + + /// + /// Verify that null location vector is rejected. + /// + [TestMethod] + public void Test_ParameterValidation_NullLocation() + { + Assert.Throws(() => + new MultivariateStudentT(5.0, null, new[,] { { 1.0 } })); + } + + /// + /// Verify that null scale matrix is rejected. + /// + [TestMethod] + public void Test_ParameterValidation_NullScaleMatrix() + { + Assert.Throws(() => + new MultivariateStudentT(5.0, new[] { 0.0 }, null)); + } + + /// + /// Verify that non-square scale matrix is rejected. + /// + [TestMethod] + public void Test_ParameterValidation_NonSquareMatrix() + { + Assert.Throws(() => + new MultivariateStudentT(5.0, new[] { 0.0, 0.0 }, new[,] { { 1.0, 0.0, 0.0 }, { 0.0, 1.0, 0.0 } })); + } + + /// + /// Verify that dimension mismatch between location and scale matrix is rejected. + /// + [TestMethod] + public void Test_ParameterValidation_DimensionMismatch() + { + Assert.Throws(() => + new MultivariateStudentT(5.0, new[] { 0.0 }, new[,] { { 1.0, 0.0 }, { 0.0, 1.0 } })); + } + + /// + /// Verify that non-positive-definite scale matrix is rejected. + /// + [TestMethod] + public void Test_ParameterValidation_NotPositiveDefinite() + { + Assert.Throws(() => + new MultivariateStudentT(5.0, new[] { 0.0, 0.0 }, new[,] { { 1.0, 2.0 }, { 2.0, 1.0 } })); + } + + /// + /// Verify that ValidateParameters returns exception instead of throwing when throwException=false. + /// + [TestMethod] + public void Test_ValidateParameters_ReturnException() + { + var mvt = CreateStandard2D(); + var ex = mvt.ValidateParameters(-1.0, new[] { 0.0 }, new[,] { { 1.0 } }, false); + Assert.IsNotNull(ex); + + ex = mvt.ValidateParameters(5.0, new[] { 0.0 }, new[,] { { 1.0 } }, false); + Assert.IsNull(ex); + } + + #endregion + + #region Clone Tests + + /// + /// Verify that Clone creates an independent deep copy. + /// + [TestMethod] + public void Test_Clone() + { + var original = CreateStandard2D(); + var clone = (MultivariateStudentT)original.Clone(); + + // Verify properties match + Assert.AreEqual(original.DegreesOfFreedom, clone.DegreesOfFreedom, 1E-10); + Assert.AreEqual(original.Dimension, clone.Dimension); + + for (int i = 0; i < original.Dimension; i++) + { + Assert.AreEqual(original.Location[i], clone.Location[i], 1E-10); + } + + // Verify PDF matches + var x = new[] { 0.5, 3.0 }; + Assert.AreEqual(original.PDF(x), clone.PDF(x), 1E-14); + + // Verify independence: modify clone parameters + clone.SetParameters(10.0, new[] { 0.0, 0.0 }, new[,] { { 2.0, 0.0 }, { 0.0, 2.0 } }); + Assert.AreEqual(5.0, original.DegreesOfFreedom, 1E-10); + Assert.AreEqual(1.0, original.Location[0], 1E-10); + } + + #endregion + + #region Constructor Tests + + /// + /// Verify the dimension-only constructor creates standard MVT (zero mean, identity scale). + /// + [TestMethod] + public void Test_Constructor_DimensionOnly() + { + var mvt = new MultivariateStudentT(3, 5.0); + + Assert.AreEqual(3, mvt.Dimension); + Assert.AreEqual(5.0, mvt.DegreesOfFreedom, 1E-10); + + // Location should be zero + for (int i = 0; i < 3; i++) + Assert.AreEqual(0.0, mvt.Location[i], 1E-10); + + // Scale matrix should be identity + var scale = mvt.ScaleMatrix; + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + Assert.AreEqual(i == j ? 1.0 : 0.0, scale[i, j], 1E-10); + } + + /// + /// Verify the location-only constructor creates MVT with identity scale. + /// + [TestMethod] + public void Test_Constructor_LocationOnly() + { + var mvt = new MultivariateStudentT(10.0, new[] { 1.0, 2.0, 3.0 }); + + Assert.AreEqual(3, mvt.Dimension); + Assert.AreEqual(10.0, mvt.DegreesOfFreedom, 1E-10); + Assert.AreEqual(1.0, mvt.Location[0], 1E-10); + Assert.AreEqual(2.0, mvt.Location[1], 1E-10); + Assert.AreEqual(3.0, mvt.Location[2], 1E-10); + } + + #endregion + + #region 3D Tests + + /// + /// Verify PDF for a 3D MVT with identity scale matrix at the mode. + /// + [TestMethod] + public void Test_PDF_3D_AtMode() + { + double nu = 10.0; + var location = new[] { 0.0, 0.0, 0.0 }; + var mvt = new MultivariateStudentT(nu, location); + + // PDF at mode: C = Γ((10+3)/2) / [Γ(10/2) · (10π)^(3/2) · |I|^(1/2)] + // = Γ(6.5) / [Γ(5) · (10π)^1.5] + // Γ(6.5) = 5.5·4.5·3.5·2.5·1.5·0.5·√π = 287.885 (approx) + // Γ(5) = 4! = 24 + double expected = Math.Exp( + Numerics.Mathematics.SpecialFunctions.Gamma.LogGamma(6.5) - + Numerics.Mathematics.SpecialFunctions.Gamma.LogGamma(5.0) - + 1.5 * Math.Log(10.0 * Math.PI)); + + double pdf = mvt.PDF(location); + Assert.AreEqual(expected, pdf, 1E-10); + } + + #endregion + + #region Non-Integer DF Tests + + /// + /// Verify that non-integer degrees of freedom (e.g., ν=4.5) works correctly. + /// This is important for Cohn's effective DF which can be non-integer. + /// + [TestMethod] + public void Test_NonIntegerDF() + { + double nu = 4.5; + var mvt = new MultivariateStudentT(nu, new[] { 0.0, 0.0 }, new[,] { { 1.0, 0.0 }, { 0.0, 1.0 } }); + + // PDF at mode should be valid + double pdfMode = mvt.PDF(new[] { 0.0, 0.0 }); + Assert.IsGreaterThan(0, pdfMode, "PDF at mode should be positive"); + Assert.IsFalse(double.IsNaN(pdfMode), "PDF at mode should not be NaN"); + + // Sampling should work + var samples = mvt.GenerateRandomValues(1000, 12345); + Assert.AreEqual(1000, samples.GetLength(0)); + Assert.AreEqual(2, samples.GetLength(1)); + + // All samples should be finite + for (int i = 0; i < 1000; i++) + { + Assert.IsFalse(double.IsNaN(samples[i, 0]), $"NaN at sample [{i}, 0]"); + Assert.IsFalse(double.IsNaN(samples[i, 1]), $"NaN at sample [{i}, 1]"); + Assert.IsFalse(double.IsInfinity(samples[i, 0]), $"Inf at sample [{i}, 0]"); + Assert.IsFalse(double.IsInfinity(samples[i, 1]), $"Inf at sample [{i}, 1]"); + } + + // Variance should be ν/(ν-2) = 4.5/2.5 = 1.8 + Assert.AreEqual(1.8, mvt.Variance[0], 1E-10); + Assert.AreEqual(1.8, mvt.Variance[1], 1E-10); + } + + /// + /// Verify that LHS sampling with non-integer DF produces correct statistics. + /// + [TestMethod] + public void Test_NonIntegerDF_LHS_Statistics() + { + double nu = 7.5; + var mvt = new MultivariateStudentT(nu, new[] { 2.0 }, new[,] { { 3.0 } }); + int N = 50000; + var samples = mvt.LatinHypercubeRandomValues(N, 99999); + + double mean = 0; + for (int i = 0; i < N; i++) + mean += samples[i, 0]; + mean /= N; + + double var_ = 0; + for (int i = 0; i < N; i++) + var_ += (samples[i, 0] - mean) * (samples[i, 0] - mean); + var_ /= (N - 1); + + // E[X] = 2.0 + Assert.AreEqual(2.0, mean, 0.05, $"Sample mean = {mean:F4}"); + + // Var[X] = ν/(ν-2) · σ² = 7.5/5.5 · 3.0 = 4.0909 + double expectedVar = (nu / (nu - 2.0)) * 3.0; + Assert.AreEqual(expectedVar, var_, 0.2, $"Sample var = {var_:F4}, expected = {expectedVar:F4}"); + } + + #endregion + + #region InverseCDF Tests + + /// + /// Verify that InverseCDF produces a deterministic, reproducible sample point. + /// + [TestMethod] + public void Test_InverseCDF_Deterministic() + { + var mvt = CreateStandard2D(); + + // Fixed probabilities: 2 for normal dims + 1 for χ² + var probs = new[] { 0.5, 0.5, 0.5 }; + + double[] x1 = mvt.InverseCDF(probs); + double[] x2 = mvt.InverseCDF(probs); + + // Same input → same output + Assert.AreEqual(x1[0], x2[0], 1E-14); + Assert.AreEqual(x1[1], x2[1], 1E-14); + } + + /// + /// Verify that InverseCDF at median probabilities (all 0.5) returns the location vector. + /// When all normal probabilities are 0.5, z = 0 for each dimension, so x = μ regardless of χ². + /// + [TestMethod] + public void Test_InverseCDF_AtMedian() + { + var mvt = CreateStandard2D(); + + // All normal probs = 0.5 → z = (0, 0). The χ² prob doesn't matter since L·0 = 0. + var probs1 = new[] { 0.5, 0.5, 0.3 }; + var probs2 = new[] { 0.5, 0.5, 0.9 }; + + double[] x1 = mvt.InverseCDF(probs1); + double[] x2 = mvt.InverseCDF(probs2); + + // Both should return exactly μ = (1, 2) + Assert.AreEqual(1.0, x1[0], 1E-10); + Assert.AreEqual(2.0, x1[1], 1E-10); + Assert.AreEqual(1.0, x2[0], 1E-10); + Assert.AreEqual(2.0, x2[1], 1E-10); + } + + /// + /// Verify that InverseCDF throws for wrong-length probability array. + /// + [TestMethod] + public void Test_InverseCDF_WrongLength() + { + var mvt = CreateStandard2D(); + + // Dimension = 2, so needs 3 probabilities + Assert.Throws(() => + mvt.InverseCDF(new[] { 0.5, 0.5 })); // too few + + Assert.Throws(() => + mvt.InverseCDF(new[] { 0.5, 0.5, 0.5, 0.5 })); // too many + } + + /// + /// Verify that small χ² probability produces more extreme samples (heavier tails). + /// When the χ² probability is near 0, W is small, so √(ν/W) is large, amplifying z. + /// + [TestMethod] + public void Test_InverseCDF_TailBehavior() + { + var mvt = new MultivariateStudentT(5.0, new[] { 0.0 }, new[,] { { 1.0 } }); + + // Fixed normal prob = 0.975 (z ≈ 1.96) + double normalProb = 0.975; + + // Small χ² prob → large scaling → more extreme sample + double[] xSmallChi = mvt.InverseCDF(new[] { normalProb, 0.01 }); + // Large χ² prob → small scaling → closer to normal + double[] xLargeChi = mvt.InverseCDF(new[] { normalProb, 0.99 }); + + Assert.IsGreaterThan(Math.Abs(xLargeChi[0]), Math.Abs(xSmallChi[0]), + $"|x_small_chi| = {Math.Abs(xSmallChi[0]):F4} should be > |x_large_chi| = {Math.Abs(xLargeChi[0]):F4}"); + } + + /// + /// Verify that InverseCDF-generated samples produce correct marginal statistics. + /// Feed uniform random probabilities through InverseCDF and check mean and variance. + /// + [TestMethod] + public void Test_InverseCDF_MarginalStatistics() + { + double nu = 5.0; + var mvt = new MultivariateStudentT(nu, new[] { 1.0, 2.0 }, new[,] { { 1.0, 0.5 }, { 0.5, 2.0 } }); + + int N = 50000; + var rnd = new MersenneTwister(42); + double mean0 = 0, mean1 = 0; + var samples0 = new double[N]; + var samples1 = new double[N]; + + for (int i = 0; i < N; i++) + { + var probs = new[] { rnd.NextDouble(), rnd.NextDouble(), rnd.NextDouble() }; + var x = mvt.InverseCDF(probs); + samples0[i] = x[0]; + samples1[i] = x[1]; + mean0 += x[0]; + mean1 += x[1]; + } + mean0 /= N; + mean1 /= N; + + // E[X] = μ + Assert.AreEqual(1.0, mean0, 0.05, $"InverseCDF mean[0] = {mean0:F4}"); + Assert.AreEqual(2.0, mean1, 0.05, $"InverseCDF mean[1] = {mean1:F4}"); + + // Var[X] = ν/(ν-2) · diag(Σ) + double var0 = 0, var1 = 0; + for (int i = 0; i < N; i++) + { + var0 += (samples0[i] - mean0) * (samples0[i] - mean0); + var1 += (samples1[i] - mean1) * (samples1[i] - mean1); + } + var0 /= (N - 1); + var1 /= (N - 1); + + Assert.AreEqual(5.0 / 3.0, var0, 0.15, $"InverseCDF Var[0] = {var0:F4}"); + Assert.AreEqual(10.0 / 3.0, var1, 0.3, $"InverseCDF Var[1] = {var1:F4}"); + } + + /// + /// Verify that InverseCDF matches GenerateRandomValues for the same uniform inputs. + /// Both methods should produce identical results given the same underlying uniforms. + /// + [TestMethod] + public void Test_InverseCDF_ConsistentWithSampling() + { + var mvt = CreateStandard2D(); + + // Generate a sample via GenerateRandomValues with known seed + var rnd = new MersenneTwister(777); + double u0 = rnd.NextDouble(); + double u1 = rnd.NextDouble(); + double u2 = rnd.NextDouble(); + + // InverseCDF with the same uniforms + double[] xInv = mvt.InverseCDF(new[] { u0, u1, u2 }); + + // GenerateRandomValues uses the same sequence internally + var samples = mvt.GenerateRandomValues(1, 777); + + Assert.AreEqual(samples[0, 0], xInv[0], 1E-10, + $"InverseCDF[0] = {xInv[0]:F6}, GenerateRandomValues[0] = {samples[0, 0]:F6}"); + Assert.AreEqual(samples[0, 1], xInv[1], 1E-10, + $"InverseCDF[1] = {xInv[1]:F6}, GenerateRandomValues[1] = {samples[0, 1]:F6}"); + } + + #endregion + + } +} \ No newline at end of file diff --git a/Test_Numerics/Distributions/Univariate/Test_Bernoulli.cs b/Test_Numerics/Distributions/Univariate/Test_Bernoulli.cs index 8e59e0a9..7259b577 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Bernoulli.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Bernoulli.cs @@ -75,16 +75,16 @@ public void Test_Bernoulli_AtRisk() double true_icdf05 = 0.0d; double true_icdf95 = 1.0d; var B = new Bernoulli(0.7d); - Assert.AreEqual(B.Mean, true_mean, 0.0001d); - Assert.AreEqual(B.Median, true_median, 0.0001d); - Assert.AreEqual(B.Mode, true_mode, 0.0001d); - Assert.AreEqual(B.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(B.Skewness, true_skew, 0.0001d); - Assert.AreEqual(B.Kurtosis, true_kurt, 0.0001d); - Assert.AreEqual(B.PDF(0.0d), true_pdf, 0.0001d); - Assert.AreEqual(B.CDF(0.5d), true_cdf, 0.0001d); - Assert.AreEqual(B.InverseCDF(0.05d), true_icdf05, 0.0001d); - Assert.AreEqual(B.InverseCDF(0.95d), true_icdf95, 0.0001d); + Assert.AreEqual(true_mean, B.Mean, 0.0001d); + Assert.AreEqual(true_median, B.Median, 0.0001d); + Assert.AreEqual(true_mode, B.Mode, 0.0001d); + Assert.AreEqual(true_stdDev, B.StandardDeviation, 0.0001d); + Assert.AreEqual(true_skew, B.Skewness, 0.0001d); + Assert.AreEqual(true_kurt, B.Kurtosis, 0.0001d); + Assert.AreEqual(true_pdf, B.PDF(0.0d), 0.0001d); + Assert.AreEqual(true_cdf, B.CDF(0.5d), 0.0001d); + Assert.AreEqual(true_icdf05, B.InverseCDF(0.05d), 0.0001d); + Assert.AreEqual(true_icdf95, B.InverseCDF(0.95d), 0.0001d); } /// @@ -138,10 +138,10 @@ public void Test_Moments() { var dist = new Bernoulli(0.3); var mom = dist.CentralMoments(200); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -244,14 +244,16 @@ public void Test_Maximum() [TestMethod()] public void Test_Kurtosis() { + // scipy.stats.bernoulli(0).stats(moments='k') => nan var b = new Bernoulli(0); - Assert.AreEqual(double.PositiveInfinity, b.Kurtosis); + Assert.AreEqual(double.NaN, b.Kurtosis); var b2 = new Bernoulli(0.3d); Assert.AreEqual(1.761904762, b2.Kurtosis, 1e-04); + // scipy.stats.bernoulli(1).stats(moments='k') => nan var b3 = new Bernoulli(1); - Assert.AreEqual(double.PositiveInfinity,b3.Kurtosis); + Assert.AreEqual(double.NaN, b3.Kurtosis); } /// @@ -260,14 +262,16 @@ public void Test_Kurtosis() [TestMethod()] public void Test_Skewness() { + // scipy.stats.bernoulli(0).stats(moments='s') => nan var b = new Bernoulli(0d); - Assert.AreEqual(double.PositiveInfinity, b.Skewness); + Assert.AreEqual(double.NaN, b.Skewness); var b2 = new Bernoulli(0.3); Assert.AreEqual(0.8728715, b2.Skewness, 1e-04); + // scipy.stats.bernoulli(1).stats(moments='s') => nan var b3 = new Bernoulli(1); - Assert.AreEqual(double.NegativeInfinity, b3.Skewness); + Assert.AreEqual(double.NaN, b3.Skewness); } /// @@ -342,5 +346,30 @@ public void Test_InverseCDF() Assert.AreEqual(1, b4.InverseCDF(0.7)); } + + /// + /// Verify Bernoulli returns NaN for skewness/kurtosis at boundary probabilities p=0 and p=1. + /// Reference: scipy.stats.bernoulli(p).stats(moments='sk') returns nan at p=0,1. + /// + [TestMethod()] + public void Test_SkewnessKurtosis_Boundary() + { + // p=0: skewness and kurtosis are undefined (p*q = 0) + var b0 = new Bernoulli(0); + Assert.AreEqual(double.NaN, b0.Skewness); + Assert.AreEqual(double.NaN, b0.Kurtosis); + + // p=1: same + var b1 = new Bernoulli(1); + Assert.AreEqual(double.NaN, b1.Skewness); + Assert.AreEqual(double.NaN, b1.Kurtosis); + + // p=0.5: well-defined + // scipy.stats.bernoulli(0.5): skew=0.0, excess_kurt=-2.0, raw_kurt=1.0 + // Note: this library uses raw kurtosis (= excess + 3) + var b5 = new Bernoulli(0.5); + Assert.AreEqual(0.0, b5.Skewness, 1E-10); + Assert.AreEqual(1.0, b5.Kurtosis, 1E-10); + } } } diff --git a/Test_Numerics/Distributions/Univariate/Test_Beta.cs b/Test_Numerics/Distributions/Univariate/Test_Beta.cs index e646928c..b744b72e 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Beta.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Beta.cs @@ -74,17 +74,17 @@ public void Test_BetaDist() double true_icdf05 = 0.000459d; double true_icdf95 = 0.7238d; var B = new BetaDistribution(0.42d, 1.57d); - Assert.AreEqual(B.Mean, true_mean, 0.0001d); - Assert.AreEqual(B.Median, true_median, 0.0001d); - Assert.AreEqual(B.Mode, true_mode, 0.0001d); - Assert.AreEqual(B.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(B.Skewness, true_skew, 0.0001d); - Assert.AreEqual(B.Kurtosis, true_kurt, 0.0001d); - Assert.AreEqual(B.PDF(0.27d), true_pdf, 0.0001d); - Assert.AreEqual(B.CDF(0.27d), true_cdf, 0.0001d); - Assert.AreEqual(B.InverseCDF(B.CDF(0.27d)), true_icdf, 0.0001d); - Assert.AreEqual(B.InverseCDF(0.05d), true_icdf05, 0.0001d); - Assert.AreEqual(B.InverseCDF(0.95d), true_icdf95, 0.0001d); + Assert.AreEqual(true_mean, B.Mean, 0.0001d); + Assert.AreEqual(true_median, B.Median, 0.0001d); + Assert.AreEqual(true_mode, B.Mode, 0.0001d); + Assert.AreEqual(true_stdDev, B.StandardDeviation, 0.0001d); + Assert.AreEqual(true_skew, B.Skewness, 0.0001d); + Assert.AreEqual(true_kurt, B.Kurtosis, 0.0001d); + Assert.AreEqual(true_pdf, B.PDF(0.27d), 0.0001d); + Assert.AreEqual(true_cdf, B.CDF(0.27d), 0.0001d); + Assert.AreEqual(true_icdf, B.InverseCDF(B.CDF(0.27d)), 0.0001d); + Assert.AreEqual(true_icdf05, B.InverseCDF(0.05d), 0.0001d); + Assert.AreEqual(true_icdf95, B.InverseCDF(0.95d), 0.0001d); } /// @@ -171,10 +171,10 @@ public void Test_Moments() { var dist = new BetaDistribution(9, 1); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_Binomial.cs b/Test_Numerics/Distributions/Univariate/Test_Binomial.cs index 7ee8660c..e3d9e237 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Binomial.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Binomial.cs @@ -282,5 +282,25 @@ public void Test_InverseCDF() var b = new Binomial(0.3, 100); Assert.AreEqual(32, b.InverseCDF(0.7)); } + + /// + /// Verify Binomial InverseCDF correctly returns NumberOfTrials at high probabilities. + /// Reference: scipy.stats.binom.ppf(p, n, p_success) + /// + [TestMethod()] + public void Test_InverseCDF_Boundary() + { + // binom(10, 0.5).ppf(0.95) = 8 + var b1 = new Binomial(0.5, 10); + Assert.AreEqual(8, b1.InverseCDF(0.95)); + // binom(10, 0.5).ppf(0.99) = 9 + Assert.AreEqual(9, b1.InverseCDF(0.99)); + // binom(10, 0.5).ppf(1.0) = 10 (must reach NumberOfTrials) + Assert.AreEqual(10, b1.InverseCDF(1.0)); + + // binom(5, 0.3).ppf(1.0) = 5 + var b2 = new Binomial(0.3, 5); + Assert.AreEqual(5, b2.InverseCDF(1.0)); + } } } diff --git a/Test_Numerics/Distributions/Univariate/Test_BootstrapAnalysis.cs b/Test_Numerics/Distributions/Univariate/Test_BootstrapAnalysis.cs index fcd409c5..00342b7f 100644 --- a/Test_Numerics/Distributions/Univariate/Test_BootstrapAnalysis.cs +++ b/Test_Numerics/Distributions/Univariate/Test_BootstrapAnalysis.cs @@ -35,6 +35,7 @@ using Numerics.Sampling; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using static System.Reflection.Metadata.BlobBuilder; namespace Distributions.Univariate @@ -162,5 +163,55 @@ public void Test_BCaCI() } + /// + /// This test verifies that UncertaintyAnalysisResults produces equivalent results to BootstrapAnalysis.Estimate. + /// + [TestMethod] + public void Test_BootstrapAnalysis_UncertaintyAnalysisResults_Equivalence() + { + var probabilities = new double[] { 0.999, 0.998, 0.995, 0.99, 0.98, 0.95, 0.9, 0.8, 0.7, 0.5, 0.3, 0.2, 0.1, 0.05, 0.02, 0.01 }; + double alpha = 0.1; + var dist = new Normal(3.122599, 0.5573654); + var boot = new BootstrapAnalysis(dist, ParameterEstimationMethod.MethodOfMoments, 100); + + // Generate bootstrap distributions once and share between both methods + var bootDistributions = boot.Distributions(); + + // Reference result from BootstrapAnalysis.Estimate + var reference = boot.Estimate(probabilities, alpha, bootDistributions, recordParameterSets: false); + + // Result from UncertaintyAnalysisResults constructor + var sampledDists = bootDistributions.Cast().ToArray(); + var result = new UncertaintyAnalysisResults(dist, sampledDists, probabilities, alpha); + + // Compare ModeCurve + Assert.HasCount(reference.ModeCurve.Length, result.ModeCurve); + for (int i = 0; i < reference.ModeCurve.Length; i++) + { + Assert.AreEqual(reference.ModeCurve[i], result.ModeCurve[i], 1E-8, + $"ModeCurve mismatch at index {i}"); + } + + // Compare ConfidenceIntervals + Assert.AreEqual(reference.ConfidenceIntervals.GetLength(0), result.ConfidenceIntervals.GetLength(0)); + Assert.AreEqual(reference.ConfidenceIntervals.GetLength(1), result.ConfidenceIntervals.GetLength(1)); + for (int i = 0; i < reference.ConfidenceIntervals.GetLength(0); i++) + { + for (int j = 0; j < reference.ConfidenceIntervals.GetLength(1); j++) + { + Assert.AreEqual(reference.ConfidenceIntervals[i, j], result.ConfidenceIntervals[i, j], 1E-8, + $"ConfidenceIntervals mismatch at [{i},{j}]"); + } + } + + // Compare MeanCurve + Assert.HasCount(reference.MeanCurve.Length, result.MeanCurve); + for (int i = 0; i < reference.MeanCurve.Length; i++) + { + Assert.AreEqual(reference.MeanCurve[i], result.MeanCurve[i], 1E-8, + $"MeanCurve mismatch at index {i}"); + } + } + } } diff --git a/Test_Numerics/Distributions/Univariate/Test_ChiSquared.cs b/Test_Numerics/Distributions/Univariate/Test_ChiSquared.cs index 5f626d9f..fbb10a5c 100644 --- a/Test_Numerics/Distributions/Univariate/Test_ChiSquared.cs +++ b/Test_Numerics/Distributions/Univariate/Test_ChiSquared.cs @@ -69,12 +69,12 @@ public void Test_ChiSquaredDist() double true_cdf = 0.49139966433823956d; double true_icdf = 6.27d; var CHI = new ChiSquared(7); - Assert.AreEqual(CHI.Mean, true_mean, 0.0001d); - Assert.AreEqual(CHI.Median, true_median, 0.0001d); - Assert.AreEqual(CHI.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(CHI.PDF(6.27d), true_pdf, 0.0001d); - Assert.AreEqual(CHI.CDF(6.27d), true_cdf, 0.0001d); - Assert.AreEqual(CHI.InverseCDF(true_cdf), true_icdf, 0.0001d); + Assert.AreEqual(true_mean, CHI.Mean, 0.0001d); + Assert.AreEqual(true_median, CHI.Median, 0.0001d); + Assert.AreEqual(true_stdDev, CHI.StandardDeviation, 0.0001d); + Assert.AreEqual(true_pdf, CHI.PDF(6.27d), 0.0001d); + Assert.AreEqual(true_cdf, CHI.CDF(6.27d), 0.0001d); + Assert.AreEqual(true_icdf, CHI.InverseCDF(true_cdf), 0.0001d); } /// @@ -125,10 +125,10 @@ public void Test_Moments() { var dist = new ChiSquared(2); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -152,7 +152,7 @@ public void Test_Median() { var x = new ChiSquared(2); var approx_median = x.DegreesOfFreedom * Math.Pow(1d - 2d / (9d * x.DegreesOfFreedom), 3d); - Assert.AreEqual(x.Median, approx_median, 1E-1); + Assert.AreEqual(approx_median, x.Median, 1E-1); } /// @@ -285,5 +285,28 @@ public void Test_InverseCDF() Assert.AreEqual(1, x2.InverseCDF(0.3934693),1e-04); Assert.AreEqual(5.5, x2.InverseCDF(0.9360721), 1e-04); } + + /// + /// Verify ChiSquared PDF does not overflow for large degrees of freedom. + /// Reference: scipy.stats.chi2.pdf(x, v) + /// + [TestMethod()] + public void Test_PDF_LargeDoF() + { + // chi2(100).pdf(100) = 0.028162503163 + var x100 = new ChiSquared(100); + Assert.AreEqual(0.028162503163, x100.PDF(100), 1E-6); + + // chi2(500).pdf(500) = 0.012611458093 + var x500 = new ChiSquared(500); + Assert.AreEqual(0.012611458093, x500.PDF(500), 1E-6); + + // chi2(1000).pdf(1000) = 0.008919133935 (should NOT be NaN or Infinity) + var x1000 = new ChiSquared(1000); + double pdf1000 = x1000.PDF(1000); + Assert.AreEqual(0.008919133935, pdf1000, 1E-6); + Assert.IsFalse(double.IsNaN(pdf1000), "PDF should not be NaN for large DoF."); + Assert.IsFalse(double.IsInfinity(pdf1000), "PDF should not be Infinity for large DoF."); + } } } diff --git a/Test_Numerics/Distributions/Univariate/Test_CompetingRisks.cs b/Test_Numerics/Distributions/Univariate/Test_CompetingRisks.cs index 2fb972fd..74104549 100644 --- a/Test_Numerics/Distributions/Univariate/Test_CompetingRisks.cs +++ b/Test_Numerics/Distributions/Univariate/Test_CompetingRisks.cs @@ -28,9 +28,12 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -using System; using Microsoft.VisualStudio.TestTools.UnitTesting; using Numerics.Distributions; +using Numerics.Mathematics; +using Numerics.Mathematics.Integration; +using Numerics.Mathematics.SpecialFunctions; +using System; namespace Distributions.Univariate { @@ -96,5 +99,944 @@ public void Test_CR_CDF() } + /// + /// Verifies that the PDF does not return Infinity or NaN when evaluated at points + /// in the left tail where CDF values approach zero under the maximum rule. + /// + /// + /// + /// Background: + /// For the maximum of independent random variables, the PDF formula involves + /// the ratio f_i(x) / F_i(x). When x is in the left tail, F_i(x) approaches zero, + /// causing division by zero if not handled properly. + /// + /// + /// Test Strategy: + /// Create a competing risks model with two Normal distributions and evaluate + /// the PDF at a point approximately 5 standard deviations below the mean, + /// where CDF values will be on the order of 1E-7. + /// + /// + /// Expected Behavior: + /// The PDF should return a small but finite non-negative value, not Infinity or NaN. + /// + /// + [TestMethod] + public void Test_PDF_MaxRule_SmallX_NoInfinity() + { + // This test verifies the fix for division by zero when CDF ≈ 0 + var dist1 = new Normal(100, 10); + var dist2 = new Normal(110, 15); + var cr = new CompetingRisks(new[] { dist1, dist2 }); + cr.MinimumOfRandomVariables = false; // Maximum rule + + // Test at a point where CDFs are very small + double x = 50; // Far in left tail + double pdf = cr.PDF(x); + + Assert.IsFalse(double.IsInfinity(pdf), "PDF should not be infinity"); + Assert.IsFalse(double.IsNaN(pdf), "PDF should not be NaN"); + Assert.IsGreaterThanOrEqualTo(0, pdf, "PDF should be non-negative"); + } + + /// + /// Verifies that the LogPDF method returns finite values for all points in a + /// randomly generated sample, ensuring numerical stability for log-likelihood calculations. + /// + /// + /// + /// Background: + /// MLE and Bayesian estimation methods work with log-likelihoods rather than + /// likelihoods to prevent numerical underflow. The LogPDF method must return + /// finite values (not NaN or +Infinity) for all valid inputs to ensure that + /// optimization algorithms can compute gradients and evaluate objective functions. + /// + /// + /// Test Strategy: + /// + /// Create a competing risks model with two Exponential distributions + /// Generate a random sample of 100 values + /// Verify LogPDF is finite for each sample point + /// Verify the sum (log-likelihood) is also finite + /// + /// + /// + /// Expected Behavior: + /// All LogPDF values should be finite negative numbers (since PDF ≤ 1 for + /// continuous distributions with unbounded support), and their sum should + /// be a finite negative number representing the log-likelihood. + /// + /// + [TestMethod] + public void Test_LogPDF_StableForMLE() + { + var dist1 = new Exponential(0.1); + var dist2 = new Exponential(0.2); + var cr = new CompetingRisks(new[] { dist1, dist2 }); + + // Generate sample + var sample = cr.GenerateRandomValues(100, 12345); + + // Verify log-likelihood is finite + double logLik = 0; + foreach (var x in sample) + { + double logPdf = cr.LogPDF(x); + Assert.IsFalse(double.IsNaN(logPdf), $"LogPDF should not be NaN at x={x}"); + Assert.IsFalse(double.IsPositiveInfinity(logPdf), $"LogPDF should not be +Inf at x={x}"); + logLik += logPdf; + } + + Assert.IsFalse(double.IsNaN(logLik), "Log-likelihood should not be NaN"); + } + + /// + /// Verifies that MLE converges to approximately correct parameter values for a + /// competing risks model under the minimum rule (series system). + /// + /// + /// + /// Background: + /// Under the minimum rule, the competing risks model represents a series system + /// where the observed outcome is the minimum of the competing random variables. + /// This is commonly used in reliability analysis where system failure occurs + /// when the first component fails. + /// + /// + /// Test Strategy: + /// + /// Create a "true" model with known Weibull parameters: + /// + /// Distribution 1: Weibull(shape=2.0, scale=100) + /// Distribution 2: Weibull(shape=3.0, scale=120) + /// + /// + /// Generate a sample of 500 observations from the true model + /// Fit a new competing risks model using MLE + /// Verify estimated parameters are within 20% of true values + /// + /// + /// + /// Expected Behavior: + /// The MLE should converge to parameter estimates close to the true values. + /// A tolerance of 20% accounts for sampling variability with n=500. + /// + /// + /// Note: + /// The minimum rule is generally more numerically stable than the maximum rule + /// because the survival function S(x) = 1 - F(x) is bounded away from zero + /// in the left tail where most data typically falls. + /// + /// + [TestMethod] + public void Test_MLE_ConvergesForMinRule() + { + // True parameters + var trueDist1 = new Weibull(2.0, 100); + var trueDist2 = new Weibull(3.0, 120); + var trueCR = new CompetingRisks(new[] { trueDist1, trueDist2 }); + trueCR.MinimumOfRandomVariables = true; + + // Generate sample + var sample = trueCR.GenerateRandomValues(500, 12345); + + // Fit model + var fitDist1 = new Weibull(); + var fitDist2 = new Weibull(); + var fitCR = new CompetingRisks(new[] { fitDist1, fitDist2 }); + fitCR.MinimumOfRandomVariables = true; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + // Verify parameters are reasonable (within 20% of true) + Assert.AreEqual(2.0, mleParams[0], 0.4, "Shape1 should be close to 2.0"); + Assert.AreEqual(100.0, mleParams[1], 20.0, "Scale1 should be close to 100"); + } + + /// + /// Verifies that MLE converges to valid (non-NaN) parameter values for a + /// competing risks model under the maximum rule (parallel system). + /// + /// + /// + /// Background: + /// Under the maximum rule, the competing risks model represents a parallel system + /// where the observed outcome is the maximum of the competing random variables. + /// This is used in reliability analysis where system failure occurs only when + /// all components have failed (redundant systems). + /// + /// + /// Test Strategy: + /// + /// Create a "true" model with known Weibull parameters: + /// + /// Distribution 1: Weibull(shape=2.0, scale=100) + /// Distribution 2: Weibull(shape=3.0, scale=120) + /// + /// + /// Generate a sample of 500 observations from the true model + /// Fit a new competing risks model using MLE + /// Verify estimated parameters are not NaN + /// + /// + /// + /// Expected Behavior: + /// The MLE should converge without numerical failures. This test uses a weaker + /// assertion (not NaN) rather than checking closeness to true values because + /// the maximum rule has inherent identifiability challenges - with only the + /// maximum observed, distinguishing between component distributions is difficult. + /// + /// + /// Note: + /// The maximum rule is more prone to numerical instability because the CDF F(x) + /// approaches zero in the left tail, causing division issues in the PDF formula. + /// This test specifically validates that the numerical stability improvements + /// allow the optimizer to complete without NaN propagation. + /// + /// + [TestMethod] + public void Test_MLE_ConvergesForMaxRule() + { + // True parameters + var trueDist1 = new Weibull(2.0, 100); + var trueDist2 = new Weibull(3.0, 120); + var trueCR = new CompetingRisks(new[] { trueDist1, trueDist2 }); + trueCR.MinimumOfRandomVariables = false; // Maximum rule + + // Generate sample + var sample = trueCR.GenerateRandomValues(500, 12345); + + // Fit model + var fitDist1 = new Weibull(); + var fitDist2 = new Weibull(); + var fitCR = new CompetingRisks(new[] { fitDist1, fitDist2 }); + fitCR.MinimumOfRandomVariables = false; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + // Verify parameters are reasonable + Assert.IsFalse(mleParams.Any(p => double.IsNaN(p)), "MLE parameters should not be NaN"); + } + + /// + /// Verifies that the PDF integrates to approximately 1 over the support of the distribution. + /// + /// + /// + /// Background: + /// A valid probability density function must integrate to 1 over its support. + /// This test verifies that the PDF implementation satisfies this fundamental + /// requirement for both minimum and maximum rules. + /// + /// + /// Test Strategy: + /// Use numerical integration (e.g., adaptive quadrature) to compute the integral + /// of the PDF from the 1E-10 quantile to the 1-1E-10 quantile, which should + /// capture essentially all of the probability mass. + /// + /// + /// Expected Behavior: + /// The integral should be within 0.001 of 1.0. + /// + /// + [TestMethod] + public void Test_PDF_IntegratesToOne() + { + var dist1 = new Normal(100, 10); + var dist2 = new Normal(110, 15); + + // Test minimum rule + var crMin = new CompetingRisks(new[] { dist1.Clone(), dist2.Clone() }); + crMin.MinimumOfRandomVariables = true; + + double lowerMin = crMin.InverseCDF(1E-10); + double upperMin = crMin.InverseCDF(1 - 1E-10); + var agkMin = new AdaptiveGaussKronrod(crMin.PDF, lowerMin, upperMin); + agkMin.Integrate(); + double integralMin = agkMin.Result; + Assert.AreEqual(1.0, integralMin, 0.001, "PDF (min rule) should integrate to 1"); + + // Test maximum rule + var crMax = new CompetingRisks(new[] { dist1.Clone(), dist2.Clone() }); + crMax.MinimumOfRandomVariables = false; + + double lowerMax = crMax.InverseCDF(1E-10); + double upperMax = crMax.InverseCDF(1 - 1E-10); + var agkMax = new AdaptiveGaussKronrod(crMax.PDF, lowerMax, upperMax); + agkMax.Integrate(); + double integralMax = agkMax.Result; + Assert.AreEqual(1.0, integralMax, 0.001, "PDF (max rule) should integrate to 1"); + } + + /// + /// Verifies consistency between PDF and CDF by checking that the numerical + /// derivative of the CDF equals the PDF at multiple points. + /// + /// + /// + /// Background: + /// By definition, f(x) = dF(x)/dx. This test verifies internal consistency + /// between the PDF and CDF implementations, which is critical for MLE where + /// both functions may be used. + /// + /// + /// Test Strategy: + /// Evaluate both the PDF and the numerical derivative of the CDF at several + /// quantile points (0.1, 0.25, 0.5, 0.75, 0.9) and verify they match within + /// numerical tolerance. + /// + /// + /// Expected Behavior: + /// The relative difference between PDF(x) and CDF'(x) should be less than 1E-4 + /// at all test points. + /// + /// + [TestMethod] + public void Test_PDF_CDF_Consistency() + { + var dist1 = new Exponential(0.1); + var dist2 = new Exponential(0.2); + var cr = new CompetingRisks(new[] { dist1, dist2 }); + + double[] quantiles = { 0.1, 0.25, 0.5, 0.75, 0.9 }; + + foreach (double q in quantiles) + { + double x = cr.InverseCDF(q); + double pdf = cr.PDF(x); + double cdfDerivative = NumericalDerivative.Derivative(cr.CDF, x); + + double relError = Math.Abs(pdf - cdfDerivative) / Math.Max(pdf, 1E-10); + Assert.IsLessThan(1E-4, relError, $"PDF and CDF derivative should match at quantile {q}. " + + $"PDF={pdf}, CDF'={cdfDerivative}, RelError={relError}"); + } + } + + /// + /// Verifies that LogPDF and log(PDF) return consistent values where both are numerically stable. + /// + /// + /// + /// Background: + /// The LogPDF method is implemented using log-space arithmetic for numerical stability. + /// This test verifies that LogPDF produces results consistent with log(PDF) in + /// regions where the standard PDF calculation is stable. + /// + /// + /// Test Strategy: + /// Compare LogPDF(x) with Math.Log(PDF(x)) at the median (where both should be stable) + /// for both minimum and maximum rules. + /// + /// + /// Expected Behavior: + /// The values should match within 1E-10 absolute tolerance at stable evaluation points. + /// + /// + [TestMethod] + public void Test_LogPDF_ConsistentWithPDF() + { + var dist1 = new Normal(100, 10); + var dist2 = new Normal(110, 15); + + // Test minimum rule + var crMin = new CompetingRisks(new[] { dist1.Clone(), dist2.Clone() }); + crMin.MinimumOfRandomVariables = true; + double xMin = crMin.Median; + double logPdfMin = crMin.LogPDF(xMin); + double logOfPdfMin = Math.Log(crMin.PDF(xMin)); + Assert.AreEqual(logOfPdfMin, logPdfMin, 1E-10, + "LogPDF should equal log(PDF) at median for min rule"); + + // Test maximum rule + var crMax = new CompetingRisks(new[] { dist1.Clone(), dist2.Clone() }); + crMax.MinimumOfRandomVariables = false; + double xMax = crMax.Median; + double logPdfMax = crMax.LogPDF(xMax); + double logOfPdfMax = Math.Log(crMax.PDF(xMax)); + Assert.AreEqual(logOfPdfMax, logPdfMax, 1E-10, + "LogPDF should equal log(PDF) at median for max rule"); + } + + + #region Minimum Rule - 2 Distributions + + // Tolerances - competing risks MLE is harder than single distribution MLE + private const double SHAPE_TOLERANCE_PERCENT = 0.25; // 25% relative error + private const double SCALE_TOLERANCE_PERCENT = 0.30; // 30% relative error + private const int SAMPLE_SIZE = 1000; + private const int RANDOM_SEED = 12345; + + /// + /// Tests MLE for minimum of Exponential and Weibull distributions. + /// + /// + /// + /// Configuration Rationale: + /// This is the classic "random failures + wear-out failures" model: + /// + /// Exponential(λ=0.02): Constant hazard rate, models random/early failures + /// Weibull(k=3, λ=80): Increasing hazard rate (k>1), models wear-out/aging + /// + /// + /// + /// Why This Is Identifiable: + /// The Exponential has constant hazard h(t) = λ, while the Weibull with k=3 has + /// hazard h(t) = (k/λ)(t/λ)^(k-1) which increases with t. The exponential dominates + /// early (small t) while the Weibull dominates later (large t). This creates + /// a characteristic "bathtub curve" effect that allows separation. + /// + /// + /// Expected Behavior: + /// The exponential rate parameter and Weibull shape/scale should be recoverable + /// within tolerance. The Weibull scale may have higher variance due to fewer + /// observations in the wear-out region. + /// + /// + [TestMethod] + public void Test_MLE_MinRule_2Dist_Exponential_Weibull() + { + // True parameters - designed for identifiability + // Weibull(k=1) ≡ Exponential, dominates early failures + // Weibull(k=3) has increasing hazard, dominates wear-out + // Note: Weibull(scale, shape=1) is used instead of Exponential to avoid + // the Exponential's 2-parameter (location+scale) MLE issues in competing risks context. + double trueScale1 = 50.0; // Weibull(50,1) = Exponential with mean 50 + double trueShape1 = 1.0; // Constant hazard + double trueScale2 = 80.0; // Mean ≈ 71 + double trueShape2 = 3.0; // Increasing hazard + + var trueDist1 = new Weibull(trueScale1, trueShape1); + var trueDist2 = new Weibull(trueScale2, trueShape2); + var trueCR = new CompetingRisks(new UnivariateDistributionBase[] { trueDist1, trueDist2 }); + trueCR.MinimumOfRandomVariables = true; + + // Generate sample + var sample = trueCR.GenerateRandomValues(SAMPLE_SIZE, RANDOM_SEED); + + // Verify sample statistics are reasonable + double sampleMean = sample.Average(); + double sampleMin = sample.Min(); + double sampleMax = sample.Max(); + Console.WriteLine($"Sample: n={SAMPLE_SIZE}, mean={sampleMean:F2}, min={sampleMin:F2}, max={sampleMax:F2}"); + + // Fit model + var fitDist1 = new Weibull(); + var fitDist2 = new Weibull(); + var fitCR = new CompetingRisks(new UnivariateDistributionBase[] { fitDist1, fitDist2 }); + fitCR.MinimumOfRandomVariables = true; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + // Weibull params are [scale, shape] for each component + Console.WriteLine($"True: Weibull({trueScale1},{trueShape1}), Weibull({trueScale2},{trueShape2})"); + Console.WriteLine($"Fitted: Weibull({mleParams[0]:F3},{mleParams[1]:F3}), Weibull({mleParams[2]:F3},{mleParams[3]:F3})"); + + // Assertions with tolerance + Assert.IsFalse(mleParams.Any(p => double.IsNaN(p) || double.IsInfinity(p)), "All parameters should be finite"); + + // Verify overall fit + double ksStatistic = ComputeKSStatistic(sample, fitCR); + Console.WriteLine($"KS statistic: {ksStatistic:F4}"); + Assert.IsLessThan(0.05, ksStatistic, "KS statistic should indicate good fit"); + } + + /// + /// Tests MLE for minimum of two Weibull distributions with different shapes. + /// + /// + /// + /// Configuration Rationale: + /// Two Weibulls with contrasting shapes and well-separated scales: + /// + /// Weibull(k=0.8, λ=30): Decreasing hazard (k<1), dominates very early + /// Weibull(k=3.0, λ=100): Increasing hazard (k>1), dominates later + /// + /// + /// + /// Why This Is Identifiable: + /// The k=0.8 distribution has decreasing hazard (infant mortality pattern), + /// while k=3.0 has increasing hazard (wear-out pattern). Combined with the + /// 3:1 scale ratio, the distributions contribute in clearly different time regions. + /// The first dominates the left tail, the second shapes the right tail. + /// + /// + [TestMethod] + public void Test_MLE_MinRule_2Dist_Weibull_DifferentShapes() + { + // Weibull 1: Decreasing hazard (infant mortality) + double trueShape1 = 0.8; + double trueScale1 = 30.0; + + // Weibull 2: Increasing hazard (wear-out) + double trueShape2 = 3.0; + double trueScale2 = 100.0; + + var trueDist1 = new Weibull(trueScale1, trueShape1); + var trueDist2 = new Weibull(trueScale2, trueShape2); + var trueCR = new CompetingRisks(new UnivariateDistributionBase[] { trueDist1, trueDist2 }); + trueCR.MinimumOfRandomVariables = true; + + var sample = trueCR.GenerateRandomValues(SAMPLE_SIZE, RANDOM_SEED); + + Console.WriteLine($"Sample: n={SAMPLE_SIZE}, mean={sample.Average():F2}, median={sample.OrderBy(x => x).ElementAt(SAMPLE_SIZE / 2):F2}"); + + // Fit model + var fitDist1 = new Weibull(); + var fitDist2 = new Weibull(); + var fitCR = new CompetingRisks(new UnivariateDistributionBase[] { fitDist1, fitDist2 }); + fitCR.MinimumOfRandomVariables = true; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + Console.WriteLine($"True: Weibull({trueShape1}, {trueScale1}), Weibull({trueShape2}, {trueScale2})"); + Console.WriteLine($"Fitted: Weibull({mleParams[0]:F3}, {mleParams[1]:F3}), Weibull({mleParams[2]:F3}, {mleParams[3]:F3})"); + + // Verify no NaN + Assert.IsFalse(mleParams.Any(double.IsNaN), "No parameters should be NaN"); + + // Check parameter recovery (allowing for label switching) + // Weibull params are [scale, shape], so shape indices are 1 and 3 + bool config1 = IsCloseRelative(mleParams[1], trueShape1, SHAPE_TOLERANCE_PERCENT) && + IsCloseRelative(mleParams[3], trueShape2, SHAPE_TOLERANCE_PERCENT); + bool config2 = IsCloseRelative(mleParams[1], trueShape2, SHAPE_TOLERANCE_PERCENT) && + IsCloseRelative(mleParams[3], trueShape1, SHAPE_TOLERANCE_PERCENT); + + Assert.IsTrue(config1 || config2, + "Fitted shapes should match true shapes (allowing for label switching)"); + } + + #endregion + + #region Minimum Rule - 3 Distributions + + /// + /// Tests MLE for minimum of three distributions: Exponential + two Weibulls. + /// + /// + /// + /// Configuration Rationale: + /// A three-component "bathtub curve" model: + /// + /// Weibull(k=0.7, λ=20): Decreasing hazard - infant mortality + /// Exponential(λ=0.005): Constant hazard - random failures (useful life) + /// Weibull(k=4, λ=150): Steeply increasing hazard - wear-out + /// + /// + /// + /// Why This Is Identifiable: + /// Each distribution dominates a different time region: + /// + /// Early: Weibull(0.7, 20) with its decreasing hazard + /// Middle: Exponential provides the "flat bottom" of the bathtub + /// Late: Weibull(4, 150) causes the upturn in failure rate + /// + /// The three distinct hazard behaviors create sufficient structure for identification. + /// + /// + /// Note: + /// With 5 parameters and complex interactions, this is a challenging estimation + /// problem. Larger samples (n=1500+) and relaxed tolerances are appropriate. + /// + /// + [TestMethod] + public void Test_MLE_MinRule_3Dist_BathtubCurve() + { + // Three-component bathtub curve using Weibulls only + // Weibull(scale, shape=1) ≡ Exponential, avoids location parameter MLE issues + var trueDist1 = new Weibull(20, 0.7); // scale=20, shape=0.7 Infant mortality (decreasing hazard) + var trueDist2 = new Weibull(200, 1.0); // scale=200, shape=1.0 Random failures (constant hazard, ≡ Exponential) + var trueDist3 = new Weibull(150, 4.0); // scale=150, shape=4.0 Wear-out (increasing hazard) + + var trueCR = new CompetingRisks(new UnivariateDistributionBase[] { trueDist1, trueDist2, trueDist3 }); + trueCR.MinimumOfRandomVariables = true; + + // Use larger sample for 3-distribution case + int n = 1500; + var sample = trueCR.GenerateRandomValues(n, RANDOM_SEED); + + Console.WriteLine($"Sample: n={n}, mean={sample.Average():F2}, min={sample.Min():F2}, max={sample.Max():F2}"); + + // Fit model + var fitDist1 = new Weibull(); + var fitDist2 = new Weibull(); + var fitDist3 = new Weibull(); + var fitCR = new CompetingRisks(new UnivariateDistributionBase[] { fitDist1, fitDist2, fitDist3 }); + fitCR.MinimumOfRandomVariables = true; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + // Params: Weibull[scale,shape] x 3 + Console.WriteLine($"True parameters: Weibull(20, 0.7), Weibull(200, 1.0), Weibull(150, 4.0)"); + Console.WriteLine($"Fitted parameters: Weibull({mleParams[0]:F3}, {mleParams[1]:F3}), " + + $"Weibull({mleParams[2]:F3}, {mleParams[3]:F3}), Weibull({mleParams[4]:F3}, {mleParams[5]:F3})"); + + // Verify convergence (no NaN/Inf) + Assert.IsFalse(mleParams.Any(p => double.IsNaN(p) || double.IsInfinity(p)), + "All parameters should be finite"); + + // Verify the overall distribution fit (CDF comparison) + double ksStatistic = ComputeKSStatistic(sample, fitCR); + Console.WriteLine($"KS statistic: {ksStatistic:F4}"); + Assert.IsLessThan(0.05, ksStatistic, "KS statistic should indicate good fit"); + } + + /// + /// Tests MLE for minimum of three Weibull distributions with distinct characteristics. + /// + /// + /// + /// Configuration Rationale: + /// Three Weibulls spanning the shape parameter space: + /// + /// Weibull(k=0.5, λ=15): Strongly decreasing hazard + /// Weibull(k=1.5, λ=60): Mildly increasing hazard + /// Weibull(k=4.0, λ=120): Strongly increasing hazard + /// + /// + /// + /// Why This Is Identifiable: + /// The shapes span k < 1, 1 < k < 2, and k > 2, giving three distinct + /// hazard behaviors. The scales are chosen to create overlapping but distinguishable + /// contributions: the k=0.5 dominates the extreme left tail, k=1.5 the middle-left, + /// and k=4.0 shapes the right tail. + /// + /// + [TestMethod] + public void Test_MLE_MinRule_3Dist_ThreeWeibulls() + { + var trueDist1 = new Weibull(15, 0.5); // Strongly decreasing hazard + var trueDist2 = new Weibull(60, 1.5); // Mildly increasing hazard + var trueDist3 = new Weibull(120, 4.0); // Strongly increasing hazard + + var trueCR = new CompetingRisks(new UnivariateDistributionBase[] { trueDist1, trueDist2, trueDist3 }); + trueCR.MinimumOfRandomVariables = true; + + int n = 1500; + var sample = trueCR.GenerateRandomValues(n, RANDOM_SEED); + + // Fit model + var fitDist1 = new Weibull(); + var fitDist2 = new Weibull(); + var fitDist3 = new Weibull(); + var fitCR = new CompetingRisks(new UnivariateDistributionBase[] { fitDist1, fitDist2, fitDist3 }); + fitCR.MinimumOfRandomVariables = true; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + Console.WriteLine($"True: Weibull(0.5, 15), Weibull(1.5, 60), Weibull(4.0, 120)"); + Console.WriteLine($"Fitted: Weibull({mleParams[0]:F2}, {mleParams[1]:F2}), " + + $"Weibull({mleParams[2]:F2}, {mleParams[3]:F2}), " + + $"Weibull({mleParams[4]:F2}, {mleParams[5]:F2})"); + + Assert.IsFalse(mleParams.Any(p => double.IsNaN(p) || double.IsInfinity(p)), + "All parameters should be finite"); + + // Verify overall fit quality + double ksStatistic = ComputeKSStatistic(sample, fitCR); + Console.WriteLine($"KS statistic: {ksStatistic:F4}"); + Assert.IsLessThan(0.05, ksStatistic, "KS statistic should indicate good fit"); + } + + #endregion + + #region Maximum Rule - 2 Distributions + + /// + /// Tests MLE for maximum of two Normal distributions with separated means. + /// + /// + /// + /// Configuration Rationale: + /// Two Normals with well-separated means and different standard deviations: + /// + /// Normal(μ=50, σ=8): Lower component + /// Normal(μ=85, σ=12): Upper component with larger spread + /// + /// + /// + /// Why This Is Identifiable: + /// For the maximum of two Normals, the lower distribution primarily influences + /// the left tail of the max distribution (when both draws happen to be low), + /// while the upper distribution dominates the right tail. With a ~4σ separation + /// between means, the contributions are clearly distinguishable. + /// + /// + /// Note: + /// Normal distributions work well for the maximum rule because their symmetric, + /// bounded tails avoid the numerical issues that arise with heavy-tailed distributions. + /// + /// + [TestMethod] + public void Test_MLE_MaxRule_2Dist_TwoNormals() + { + double trueMu1 = 50, trueSigma1 = 8; + double trueMu2 = 85, trueSigma2 = 12; + + var trueDist1 = new Normal(trueMu1, trueSigma1); + var trueDist2 = new Normal(trueMu2, trueSigma2); + var trueCR = new CompetingRisks(new UnivariateDistributionBase[] { trueDist1, trueDist2 }); + trueCR.MinimumOfRandomVariables = false; // Maximum rule + + var sample = trueCR.GenerateRandomValues(SAMPLE_SIZE, RANDOM_SEED); + + Console.WriteLine($"Sample: n={SAMPLE_SIZE}, mean={sample.Average():F2}, min={sample.Min():F2}, max={sample.Max():F2}"); + + // Fit model + var fitDist1 = new Normal(); + var fitDist2 = new Normal(); + var fitCR = new CompetingRisks(new UnivariateDistributionBase[] { fitDist1, fitDist2 }); + fitCR.MinimumOfRandomVariables = false; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + Console.WriteLine($"True: Normal({trueMu1}, {trueSigma1}), Normal({trueMu2}, {trueSigma2})"); + Console.WriteLine($"Fitted: Normal({mleParams[0]:F2}, {mleParams[1]:F2}), Normal({mleParams[2]:F2}, {mleParams[3]:F2})"); + + // Verify no NaN + Assert.IsFalse(mleParams.Any(double.IsNaN), "No parameters should be NaN"); + + // Check overall fit quality + double ksStatistic = ComputeKSStatistic(sample, fitCR); + Console.WriteLine($"KS statistic: {ksStatistic:F4}"); + Assert.IsLessThan(0.05, ksStatistic, "KS statistic should indicate good fit"); + + // Verify means are approximately recovered (allowing label switching) + var fittedMeans = new[] { mleParams[0], mleParams[2] }.OrderBy(x => x).ToArray(); + var trueMeans = new[] { trueMu1, trueMu2 }.OrderBy(x => x).ToArray(); + + Assert.IsTrue(IsCloseRelative(fittedMeans[0], trueMeans[0], SCALE_TOLERANCE_PERCENT), + $"Lower mean should be close to {trueMeans[0]}"); + Assert.IsTrue(IsCloseRelative(fittedMeans[1], trueMeans[1], SCALE_TOLERANCE_PERCENT), + $"Upper mean should be close to {trueMeans[1]}"); + } + + /// + /// Tests MLE for maximum of Weibull and Gumbel (GEV Type I) distributions. + /// + /// + /// + /// Configuration Rationale: + /// Combining distributions with different tail behaviors: + /// + /// Weibull(k=2, λ=50): Light right tail (bounded support if k>1 effective tail) + /// Gumbel(μ=70, σ=15): Heavy right tail (extreme value distribution) + /// + /// + /// + /// Why This Is Identifiable: + /// For the maximum, the right tail behavior is critical. The Weibull's relatively + /// light tail means it rarely produces extreme maxima, while the Gumbel's heavy + /// tail dominates extreme values. The bulk of the distribution is shaped by both, + /// with the Gumbel's influence increasing in the upper quantiles. + /// + /// + [TestMethod] + public void Test_MLE_MaxRule_2Dist_Weibull_Gumbel() + { + double trueWeibullShape = 2.0; + double trueWeibullScale = 50.0; + double trueGumbelLocation = 70.0; + double trueGumbelScale = 15.0; + + var trueDist1 = new Weibull(trueWeibullScale, trueWeibullShape); + var trueDist2 = new Gumbel(trueGumbelLocation, trueGumbelScale); + var trueCR = new CompetingRisks(new UnivariateDistributionBase[] { trueDist1, trueDist2 }); + trueCR.MinimumOfRandomVariables = false; // Maximum rule + + var sample = trueCR.GenerateRandomValues(SAMPLE_SIZE, RANDOM_SEED); + + Console.WriteLine($"Sample: n={SAMPLE_SIZE}, mean={sample.Average():F2}, min={sample.Min():F2}, max={sample.Max():F2}"); + + // Fit model + var fitDist1 = new Weibull(); + var fitDist2 = new Gumbel(); + var fitCR = new CompetingRisks(new UnivariateDistributionBase[] { fitDist1, fitDist2 }); + fitCR.MinimumOfRandomVariables = false; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + Console.WriteLine($"True: Weibull({trueWeibullShape}, {trueWeibullScale}), Gumbel({trueGumbelLocation}, {trueGumbelScale})"); + Console.WriteLine($"Fitted: Weibull({mleParams[0]:F2}, {mleParams[1]:F2}), Gumbel({mleParams[2]:F2}, {mleParams[3]:F2})"); + + Assert.IsFalse(mleParams.Any(double.IsNaN), "No parameters should be NaN"); + + // Verify overall fit + double ksStatistic = ComputeKSStatistic(sample, fitCR); + Console.WriteLine($"KS statistic: {ksStatistic:F4}"); + Assert.IsLessThan(0.05, ksStatistic, "KS statistic should indicate good fit"); + } + + #endregion + + #region Maximum Rule - 3 Distributions + + /// + /// Tests MLE for maximum of three Normal distributions representing a trimodal scenario. + /// + /// + /// + /// Configuration Rationale: + /// Three well-separated Normals: + /// + /// Normal(μ=40, σ=6): Low component + /// Normal(μ=70, σ=8): Middle component + /// Normal(μ=100, σ=10): High component + /// + /// + /// + /// Why This Is Identifiable: + /// With ~4σ separation between adjacent means, each Normal dominates a distinct + /// region. For the maximum, the observed values come from all three components + /// but with different frequencies based on the probability that each component + /// produces the largest value. The lowest component rarely "wins" but influences + /// the extreme left tail of the maximum distribution. + /// + /// + [TestMethod] + public void Test_MLE_MaxRule_3Dist_ThreeNormals() + { + var trueDist1 = new Normal(40, 6); + var trueDist2 = new Normal(70, 8); + var trueDist3 = new Normal(100, 10); + + var trueCR = new CompetingRisks(new UnivariateDistributionBase[] { trueDist1, trueDist2, trueDist3 }); + trueCR.MinimumOfRandomVariables = false; // Maximum rule + + int n = 1500; + var sample = trueCR.GenerateRandomValues(n, RANDOM_SEED); + + Console.WriteLine($"Sample: n={n}, mean={sample.Average():F2}, min={sample.Min():F2}, max={sample.Max():F2}"); + + // Fit model + var fitDist1 = new Normal(); + var fitDist2 = new Normal(); + var fitDist3 = new Normal(); + var fitCR = new CompetingRisks(new UnivariateDistributionBase[] { fitDist1, fitDist2, fitDist3 }); + fitCR.MinimumOfRandomVariables = false; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + Console.WriteLine($"True: N(40,6), N(70,8), N(100,10)"); + Console.WriteLine($"Fitted: N({mleParams[0]:F1},{mleParams[1]:F1}), " + + $"N({mleParams[2]:F1},{mleParams[3]:F1}), " + + $"N({mleParams[4]:F1},{mleParams[5]:F1})"); + + Assert.IsFalse(mleParams.Any(p => double.IsNaN(p) || double.IsInfinity(p)), + "All parameters should be finite"); + + // Verify overall fit + double ksStatistic = ComputeKSStatistic(sample, fitCR); + Console.WriteLine($"KS statistic: {ksStatistic:F4}"); + Assert.IsLessThan(0.06, ksStatistic, "KS statistic should indicate reasonable fit"); + + // Note: Individual parameter recovery is not asserted for 3 same-family components + // under max-rule due to inherent identifiability limitations. The lowest component + // has minimal influence on the maximum and is difficult to recover. + // KS statistic and convergence checks above are sufficient. + } + + /// + /// Tests MLE for maximum of Exponential, Gamma, and LogNormal - three different families. + /// + /// + /// + /// Configuration Rationale: + /// Three different distribution families with distinct shapes: + /// + /// Exponential(λ=0.05): Monotonically decreasing density, mean=20 + /// Gamma(k=3, θ=15): Unimodal with mode at 30, mean=45 + /// LogNormal(μ=4.2, σ=0.4): Right-skewed, median≈67, mean≈72 + /// + /// + /// + /// Why This Is Identifiable: + /// Using three different families provides maximum structural diversity. + /// The Exponential is memoryless, the Gamma has a characteristic shape controlled + /// by its shape parameter, and the LogNormal has a distinctive heavy right tail. + /// For the maximum, these combine to create a complex but estimable distribution. + /// + /// + [TestMethod] + public void Test_MLE_MaxRule_3Dist_DifferentFamilies() + { + var trueDist1 = new Exponential(0.05); // Mean = 20 + var trueDist2 = new GammaDistribution(3.0, 15.0); // Mean = 45 + var trueDist3 = new LogNormal(4.2, 0.4) { Base = Math.E }; // Median ≈ 67 + + var trueCR = new CompetingRisks(new UnivariateDistributionBase[] { trueDist1, trueDist2, trueDist3 }); + trueCR.MinimumOfRandomVariables = false; // Maximum rule + + int n = 1500; + var sample = trueCR.GenerateRandomValues(n, RANDOM_SEED); + + Console.WriteLine($"Sample: n={n}, mean={sample.Average():F2}, min={sample.Min():F2}, max={sample.Max():F2}"); + Console.WriteLine($"True distribution means: Exp={1 / 0.05:F0}, Gamma={3 * 15:F0}, LogN≈{Math.Exp(4.2 + 0.4 * 0.4 / 2):F0}"); + + // Fit model + var fitDist1 = new Exponential(); + var fitDist2 = new GammaDistribution(); + var fitDist3 = new LogNormal(); + var fitCR = new CompetingRisks(new UnivariateDistributionBase[] { fitDist1, fitDist2, fitDist3 }); + fitCR.MinimumOfRandomVariables = false; + + var mleParams = fitCR.MLE(sample); + fitCR.SetParameters(mleParams); + + Console.WriteLine($"True: Exp(0.05), Gamma(3, 15), LogNormal(4.2, 0.4)"); + Console.WriteLine($"Fitted: Exp({mleParams[0]:F4}), Gamma({mleParams[1]:F2}, {mleParams[2]:F2}), " + + $"LogNormal({mleParams[3]:F2}, {mleParams[4]:F2})"); + + Assert.IsFalse(mleParams.Any(p => double.IsNaN(p) || double.IsInfinity(p)), + "All parameters should be finite"); + + // Verify overall fit + double ksStatistic = ComputeKSStatistic(sample, fitCR); + Console.WriteLine($"KS statistic: {ksStatistic:F4}"); + Assert.IsLessThan(0.06, ksStatistic, "KS statistic should indicate reasonable fit"); + } + + #endregion + + #region Helper Methods + + /// + /// Checks if two values are close within a relative tolerance. + /// + private static bool IsCloseRelative(double actual, double expected, double tolerance) + { + if (expected == 0) return Math.Abs(actual) < tolerance; + return Math.Abs(actual - expected) / Math.Abs(expected) < tolerance; + } + + /// + /// Computes the Kolmogorov-Smirnov statistic for goodness of fit. + /// + private static double ComputeKSStatistic(double[] sample, CompetingRisks distribution) + { + var sorted = sample.OrderBy(x => x).ToArray(); + int n = sorted.Length; + double maxDiff = 0; + + for (int i = 0; i < n; i++) + { + double empiricalCDF = (i + 1.0) / n; + double theoreticalCDF = distribution.CDF(sorted[i]); + double diff = Math.Abs(empiricalCDF - theoreticalCDF); + if (diff > maxDiff) maxDiff = diff; + } + + return maxDiff; + } + + #endregion } } diff --git a/Test_Numerics/Distributions/Univariate/Test_Exponential.cs b/Test_Numerics/Distributions/Univariate/Test_Exponential.cs index 4e2ca62f..f904a0ef 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Exponential.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Exponential.cs @@ -97,8 +97,8 @@ public void Test_EXP_LMOM_Fit() double a = EXP.Alpha; double true_x = 1372.333d; double true_a = 276.4731d; - Assert.AreEqual(x, true_x, 0.001d); - Assert.AreEqual(a, true_a, 0.001d); + Assert.AreEqual(true_x, x, 0.001d); + Assert.AreEqual(true_a, a, 0.001d); var lmom = EXP.LinearMomentsFromParameters(EXP.GetParameters); Assert.AreEqual(1648.806d, lmom[0], 0.001d); Assert.AreEqual(138.2366d, lmom[1], 0.001d); @@ -150,7 +150,7 @@ public void Test_EXP_Quantile() Assert.IsLessThan(0.01d, (q100 - true_100) / true_100); double p = EXP.CDF(q100); double true_p = 0.99d; - Assert.AreEqual(p, true_p); + Assert.AreEqual(true_p, p); } /// @@ -251,10 +251,10 @@ public void Test_Moments() { var dist = new Exponential(1, 1); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_GammaDistribution.cs b/Test_Numerics/Distributions/Univariate/Test_GammaDistribution.cs index eb90c7a7..62c26993 100644 --- a/Test_Numerics/Distributions/Univariate/Test_GammaDistribution.cs +++ b/Test_Numerics/Distributions/Univariate/Test_GammaDistribution.cs @@ -75,8 +75,8 @@ public void Test_GammaDist_MOM() double lambda = G.Kappa; double trueA = 0.08317d; double trueL = 15.91188d; - Assert.IsLessThan(0.01d,(alpha - trueA) / trueA); - Assert.IsLessThan(0.01d,(lambda - trueL) / trueL); + Assert.IsLessThan(0.01d, (alpha - trueA) / trueA); + Assert.IsLessThan(0.01d, (lambda - trueL) / trueL); } [TestMethod()] @@ -90,12 +90,12 @@ public void Test_GammaDist_LMOM_Fit() double shape = G.Kappa; double true_scale = 1.280143d; double true_shape = 7.778442d; - Assert.AreEqual(scale, true_scale, 0.0001d); - Assert.AreEqual(shape, true_shape, 0.0001d); + Assert.AreEqual(true_scale, scale, 0.0001d); + Assert.AreEqual(true_shape, shape, 0.0001d); var lmom = G.LinearMomentsFromParameters(G.GetParameters); - Assert.AreEqual(9.9575163d, lmom[0], 0.0001d); + Assert.AreEqual(9.9575163d, lmom[0], 0.0001d); Assert.AreEqual(1.9822363d, lmom[1], 0.0001d); - Assert.AreEqual(0.1175059d,lmom[2], 0.0001d); + Assert.AreEqual(0.1175059d, lmom[2], 0.0001d); Assert.AreEqual(0.1268391d, lmom[3], 0.0001d); } @@ -143,7 +143,7 @@ public void Test_GammaDist_Quantile() Assert.IsLessThan(0.01d, (q1000 - true_1000) / true_1000); double p = G.CDF(q1000); double true_p = 0.99d; - Assert.AreEqual(p, true_p); + Assert.AreEqual(true_p, p); } /// @@ -182,7 +182,7 @@ public void Test_Construction() { var G = new GammaDistribution(2, 10); Assert.AreEqual(2,G.Theta); - Assert.AreEqual(10,G.Kappa); + Assert.AreEqual(10, G.Kappa); var G2 = new GammaDistribution(-1, 4); Assert.AreEqual(-1,G2.Theta); @@ -197,10 +197,10 @@ public void Test_Construction() public void Test_Rate() { var G = new GammaDistribution(2, 2); - Assert.AreEqual(0.5, G.Rate); + Assert.AreEqual(0.5,G.Rate); var G2 = new GammaDistribution(); - Assert.AreEqual(0.1, G2.Rate); + Assert.AreEqual(0.1,G2.Rate); } /// @@ -210,10 +210,10 @@ public void Test_Rate() public void Test_ParametersToString() { var G = new GammaDistribution(); - Assert.AreEqual("Scale (θ)",G.ParametersToString[0, 0] ); - Assert.AreEqual("Shape (κ)", G.ParametersToString[1, 0]); - Assert.AreEqual("10", G.ParametersToString[0, 1]); - Assert.AreEqual("2", G.ParametersToString[1, 1]); + Assert.AreEqual("Scale (θ)", G.ParametersToString[0, 0]); + Assert.AreEqual("Shape (κ)",G.ParametersToString[1, 0]); + Assert.AreEqual("10",G.ParametersToString[0, 1]); + Assert.AreEqual("2",G.ParametersToString[1, 1]); } /// @@ -243,10 +243,10 @@ public void Test_Moments() { var dist = new GammaDistribution(); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -256,7 +256,7 @@ public void Test_Moments() public void Test_Mean() { var G = new GammaDistribution(); - Assert.AreEqual(20, G.Mean); + Assert.AreEqual(20,G.Mean); } /// @@ -302,7 +302,7 @@ public void Test_StandardDeviation() public void Test_Skewness() { var G = new GammaDistribution(); - Assert.AreEqual(1.4142135, G.Skewness, 1e-04); + Assert.AreEqual(1.4142135, G.Skewness, 1e-04); var G2 = new GammaDistribution(10, 100); Assert.AreEqual(0.2, G2.Skewness); @@ -321,7 +321,7 @@ public void Test_Kurtosis() Assert.AreEqual(4, G2.Kurtosis); var G3 = new GammaDistribution(10, 2.5); - Assert.AreEqual(5.4, G3.Kurtosis); + Assert.AreEqual(5.4,G3.Kurtosis); } /// @@ -331,7 +331,7 @@ public void Test_Kurtosis() public void Test_Minimum() { var G = new GammaDistribution(); - Assert.AreEqual(0, G.Minimum); + Assert.AreEqual(0,G.Minimum); } /// @@ -341,7 +341,7 @@ public void Test_Minimum() public void Test_Maximum() { var G = new GammaDistribution(); - Assert.AreEqual(double.PositiveInfinity,G.Maximum ); + Assert.AreEqual(double.PositiveInfinity,G.Maximum); } /// @@ -364,8 +364,8 @@ public void ValidateMLE_NR() double lambda = G.Kappa; double trueA = 0.08833d; double trueL = 16.89937d; - Assert.IsLessThan(0.2d,(alpha - trueA) / trueA); - Assert.IsLessThan(0.01d,(lambda - trueL) / trueL); + Assert.IsLessThan( 0.2d, (alpha - trueA) / trueA ); + Assert.IsLessThan(0.01d, (lambda - trueL) / trueL); } /// @@ -400,10 +400,10 @@ public void Test_PDF() { var G = new GammaDistribution(10,1); Assert.AreEqual(0.090483, G.PDF(1), 1e-04); - Assert.AreEqual(0.036787, G.PDF(10), 1e-04); + Assert.AreEqual(0.036787, G.PDF(10), 1e-04); var G2 = new GammaDistribution(1,1); - Assert.AreEqual(0.367879, G2.PDF(1), 1e-04); + Assert.AreEqual(0.367879, G2.PDF(1), 1e-04); Assert.AreEqual(0.0000453999, G2.PDF(10), 1e-10); } @@ -418,7 +418,7 @@ public void Test_CDF() Assert.AreEqual(0.63212, G.CDF(10), 1e-04); var G2 = new GammaDistribution(1, 1); - Assert.AreEqual(0.999954, G2.CDF(10), 1e-04); + Assert.AreEqual(0.999954, G2.CDF(10), 1e-04); var G3 = new GammaDistribution(0.1, 10); Assert.AreEqual(0.54207028, G3.CDF(1), 1e-04); diff --git a/Test_Numerics/Distributions/Univariate/Test_GeneralizedBeta.cs b/Test_Numerics/Distributions/Univariate/Test_GeneralizedBeta.cs index edb5db03..257a6ab5 100644 --- a/Test_Numerics/Distributions/Univariate/Test_GeneralizedBeta.cs +++ b/Test_Numerics/Distributions/Univariate/Test_GeneralizedBeta.cs @@ -74,13 +74,13 @@ public void Test_GenBeta() double true_cdf = 0.69358638272337991d; double true_icdf = 0.27d; - Assert.AreEqual(B.Mean, true_mean, 0.0001d); - Assert.AreEqual(B.Median, true_median, 0.0001d); - Assert.AreEqual(B.Mode, true_mode, 0.0001d); - Assert.AreEqual(B.Variance, true_var, 0.0001d); - Assert.AreEqual(B.PDF(0.27d), true_pdf, 0.0001d); - Assert.AreEqual(B.CDF(0.27d), true_cdf, 0.0001d); - Assert.AreEqual(B.InverseCDF(B.CDF(0.27d)), true_icdf, 0.0001d); + Assert.AreEqual(true_mean, B.Mean, 0.0001d); + Assert.AreEqual(true_median, B.Median, 0.0001d); + Assert.AreEqual(true_mode, B.Mode, 0.0001d); + Assert.AreEqual(true_var, B.Variance, 0.0001d); + Assert.AreEqual(true_pdf, B.PDF(0.27d), 0.0001d); + Assert.AreEqual(true_cdf, B.CDF(0.27d), 0.0001d); + Assert.AreEqual(true_icdf, B.InverseCDF(B.CDF(0.27d)), 0.0001d); } @@ -188,10 +188,10 @@ public void Test_Moments() { var dist = new GeneralizedBeta(2, 2, 0, 1); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -338,5 +338,29 @@ public void Test_InverseCDF() var b3 = new GeneralizedBeta(5, 100,0,10); Assert.AreEqual(0, b3.InverseCDF(0)); } + + /// + /// Verify GeneralizedBeta Mode for the uniform case (Alpha=Beta=1) returns the midpoint. + /// Reference: scipy.stats.beta(a, b) mode = (a-1)/(a+b-2); undefined for a=b=1 (uniform). + /// + [TestMethod()] + public void Test_Mode_UniformCase() + { + // Alpha=1, Beta=1 on [0,1] is uniform: mode should be midpoint = 0.5 + var b1 = new GeneralizedBeta(1, 1, 0, 1); + Assert.AreEqual(0.5, b1.Mode, 1E-10); + + // Alpha=2, Beta=5 on [0,1]: mode = (2-1)/(2+5-2) = 0.2 + var b2 = new GeneralizedBeta(2, 5, 0, 1); + Assert.AreEqual(0.2, b2.Mode, 1E-10); + + // Alpha=2, Beta=2 on [0,1]: mode = (2-1)/(2+2-2) = 0.5 + var b3 = new GeneralizedBeta(2, 2, 0, 1); + Assert.AreEqual(0.5, b3.Mode, 1E-10); + + // Alpha=5, Beta=2 on [0,1]: mode = (5-1)/(5+2-2) = 0.8 + var b4 = new GeneralizedBeta(5, 2, 0, 1); + Assert.AreEqual(0.8, b4.Mode, 1E-10); + } } } diff --git a/Test_Numerics/Distributions/Univariate/Test_GeneralizedExtremeValue.cs b/Test_Numerics/Distributions/Univariate/Test_GeneralizedExtremeValue.cs index ca9f0ce3..3cbb9db6 100644 --- a/Test_Numerics/Distributions/Univariate/Test_GeneralizedExtremeValue.cs +++ b/Test_Numerics/Distributions/Univariate/Test_GeneralizedExtremeValue.cs @@ -112,9 +112,9 @@ public void Test_GEV_LMOM_Fit() double true_x = 1543.933d; double true_a = 218.1148d; double true_k = 0.1068473d; - Assert.AreEqual(x, true_x, 0.001d); - Assert.AreEqual(a, true_a, 0.001d); - Assert.AreEqual(k, true_k, 0.001d); + Assert.AreEqual(true_x, x, 0.001d); + Assert.AreEqual(true_a, a, 0.001d); + Assert.AreEqual(true_k, k, 0.001d); var lmom = GEV.LinearMomentsFromParameters(GEV.GetParameters); Assert.AreEqual(1648.806d, lmom[0], 0.001d); Assert.AreEqual(138.2366d, lmom[1], 0.001d); @@ -274,10 +274,10 @@ public void Test_Moments() { var dist = new GeneralizedExtremeValue(100, 10, -0.1); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_GeneralizedLogistic.cs b/Test_Numerics/Distributions/Univariate/Test_GeneralizedLogistic.cs index d5560ded..da5b5f64 100644 --- a/Test_Numerics/Distributions/Univariate/Test_GeneralizedLogistic.cs +++ b/Test_Numerics/Distributions/Univariate/Test_GeneralizedLogistic.cs @@ -112,14 +112,14 @@ public void Test_GLO_LMOM_Fit() double true_x = 1625.42d; double true_a = 135.8186d; double true_k = -0.1033903d; - Assert.AreEqual(x, true_x, 0.001d); - Assert.AreEqual(a, true_a, 0.001d); - Assert.AreEqual(k, true_k, 0.001d); + Assert.AreEqual(true_x, x, 0.001d); + Assert.AreEqual(true_a, a, 0.001d); + Assert.AreEqual(true_k, k, 0.001d); var lmom = GLO.LinearMomentsFromParameters(GLO.GetParameters); Assert.AreEqual(1648.806d, lmom[0], 0.001d); - Assert.AreEqual(138.2366d, lmom[1], 0.001d); - Assert.AreEqual(0.1033903d, lmom[2], 0.001d); - Assert.AreEqual(0.1755746d, lmom[3], 0.001d); + Assert.AreEqual(138.2366d, lmom[1], 0.001d); + Assert.AreEqual(0.1033903d, lmom[2], 0.001d); + Assert.AreEqual(0.1755746d, lmom[3], 0.001d); } /// @@ -169,7 +169,7 @@ public void Test_GLO_Quantile() Assert.IsLessThan(0.01d, (q100 - true_100) / true_100); double p = GLO.CDF(q100); double true_p = 0.99d; - Assert.AreEqual(p, true_p); + Assert.AreEqual(true_p, p); } /// @@ -199,7 +199,7 @@ public void Test_Construction() var l = new GeneralizedLogistic(); Assert.AreEqual(100,l.Xi); Assert.AreEqual(10,l.Alpha); - Assert.AreEqual(0,l.Kappa); + Assert.AreEqual(0, l.Kappa); var l2 = new GeneralizedLogistic(-100, 10, 1); Assert.AreEqual(-100,l2.Xi); @@ -246,10 +246,10 @@ public void Test_Moments() { var dist = new GeneralizedLogistic(100, 10, -0.1); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -323,7 +323,7 @@ public void Test_Skewness() Assert.AreEqual(-10.90354, l2.Skewness, 1e-04); var l3 = new GeneralizedLogistic(100, 10, 1); - Assert.AreEqual(double.NaN, l3.Skewness); + Assert.AreEqual(double.NaN,l3.Skewness); } /// @@ -404,7 +404,7 @@ public void Test_CDF() public void Test_InverseCDF() { var l = new GeneralizedLogistic(); - Assert.AreEqual(double.NegativeInfinity,l.InverseCDF(0)); + Assert.AreEqual(double.NegativeInfinity,l.InverseCDF(0) ); Assert.AreEqual(100, l.InverseCDF(0.5)); Assert.AreEqual(double.PositiveInfinity, l.InverseCDF(1)); diff --git a/Test_Numerics/Distributions/Univariate/Test_GeneralizedNormal.cs b/Test_Numerics/Distributions/Univariate/Test_GeneralizedNormal.cs index b28d1e98..8f397e10 100644 --- a/Test_Numerics/Distributions/Univariate/Test_GeneralizedNormal.cs +++ b/Test_Numerics/Distributions/Univariate/Test_GeneralizedNormal.cs @@ -73,9 +73,9 @@ public void Test_GNO_LMOM() double true_a = 3.4885029; double true_k = -0.1307169; - Assert.AreEqual(xi, true_xi, 0.0001d); - Assert.AreEqual(a, true_a, 0.0001d); - Assert.AreEqual(k, true_k, 0.0001d); + Assert.AreEqual(true_xi, xi, 0.0001d); + Assert.AreEqual(true_a, a, 0.0001d); + Assert.AreEqual(true_k, k, 0.0001d); var lmom = gno.LinearMomentsFromParameters(gno.GetParameters); Assert.AreEqual(9.95751634, lmom[0], 0.0001d); @@ -125,9 +125,9 @@ public void Test_GNO_Dist() double true_cdf = 0.912294; double true_invcdf = 14.9; - Assert.AreEqual(pdf, true_pdf, 0.0001d); - Assert.AreEqual(cdf, true_cdf, 0.0001d); - Assert.AreEqual(invcdf, true_invcdf, 0.0001d); + Assert.AreEqual(true_pdf, pdf, 0.0001d); + Assert.AreEqual(true_cdf, cdf, 0.0001d); + Assert.AreEqual(true_invcdf, invcdf, 0.0001d); } @@ -214,10 +214,10 @@ public void Test_Moments() { var dist = new GeneralizedNormal(100, 10, -0.1); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } diff --git a/Test_Numerics/Distributions/Univariate/Test_GeneralizedPareto.cs b/Test_Numerics/Distributions/Univariate/Test_GeneralizedPareto.cs index 501e3edc..00106d42 100644 --- a/Test_Numerics/Distributions/Univariate/Test_GeneralizedPareto.cs +++ b/Test_Numerics/Distributions/Univariate/Test_GeneralizedPareto.cs @@ -104,9 +104,9 @@ public void Test_GPA_LMOM_Fit() double true_x = 1285.909d; double true_a = 589.7772d; double true_k = 0.6251903d; - Assert.AreEqual(x, true_x, 0.001d); - Assert.AreEqual(a, true_a, 0.001d); - Assert.AreEqual(k, true_k, 0.001d); + Assert.AreEqual(true_x, x, 0.001d); + Assert.AreEqual(true_a, a, 0.001d); + Assert.AreEqual(true_k, k, 0.001d); var lmom = GPA.LinearMomentsFromParameters(GPA.GetParameters); Assert.AreEqual(1648.806d, lmom[0], 0.001d); Assert.AreEqual(138.2366d, lmom[1], 0.001d); @@ -293,10 +293,10 @@ public void Test_Moments() { var dist = new GeneralizedPareto(100, 10, -0.1); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_Gumbel.cs b/Test_Numerics/Distributions/Univariate/Test_Gumbel.cs index 0c0ff506..19a72764 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Gumbel.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Gumbel.cs @@ -97,8 +97,8 @@ public void Test_GUM_LMOM_Fit() double a = GUM.Alpha; double true_x = 1533.69d; double true_a = 199.4332d; - Assert.AreEqual(x, true_x, 0.001d); - Assert.AreEqual(a, true_a, 0.001d); + Assert.AreEqual(true_x, x, 0.001d); + Assert.AreEqual(true_a, a, 0.001d); var lmom = GUM.LinearMomentsFromParameters(GUM.GetParameters); Assert.AreEqual(1648.806d, lmom[0], 0.001d); Assert.AreEqual(138.2366d, lmom[1], 0.001d); @@ -183,11 +183,11 @@ public void Test_Gumbel_StandardError() public void Test_Construction() { var GUM = new Gumbel(); - Assert.AreEqual(100,GUM.Xi); - Assert.AreEqual(10,GUM.Alpha); + Assert.AreEqual(100, GUM.Xi); + Assert.AreEqual(10, GUM.Alpha); var GUM2 = new Gumbel(-100, 1); - Assert.AreEqual(-100,GUM2.Xi); + Assert.AreEqual(-100, GUM2.Xi); Assert.AreEqual(1, GUM2.Alpha); } @@ -228,10 +228,10 @@ public void Test_Moments() { var dist = new Gumbel(10, 1); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -257,7 +257,7 @@ public void Test_Median() Assert.AreEqual(103.66512, GUM.Median, 1e-05); var GUM2 = new Gumbel(10, 1); - Assert.AreEqual(10.366512, GUM2.Median, 1e-04); + Assert.AreEqual(10.366512, GUM2.Median, 1e-04); } /// @@ -270,7 +270,7 @@ public void Test_StandardDeviation() Assert.AreEqual(12.82549, GUM.StandardDeviation, 1e-04); var GUM2 = new Gumbel(10, 1); - Assert.AreEqual(1.28254, GUM2.StandardDeviation, 1e-04); + Assert.AreEqual(1.28254, GUM2.StandardDeviation, 1e-04); } /// @@ -336,7 +336,7 @@ public void Test_PDF() public void Test_CDF() { var GUM = new Gumbel(); - Assert.AreEqual(0.36787, GUM.CDF(100), 1e-04); + Assert.AreEqual(0.36787, GUM.CDF(100), 1e-04); Assert.AreEqual(3.5073e-65, GUM.CDF(50), 1e-68); Assert.AreEqual(0,GUM.CDF(-10)); diff --git a/Test_Numerics/Distributions/Univariate/Test_InverseChiSquared.cs b/Test_Numerics/Distributions/Univariate/Test_InverseChiSquared.cs index ee0651a9..5c6d905a 100644 --- a/Test_Numerics/Distributions/Univariate/Test_InverseChiSquared.cs +++ b/Test_Numerics/Distributions/Univariate/Test_InverseChiSquared.cs @@ -60,10 +60,12 @@ public class Test_InverseChiSquared [TestMethod()] public void Test_InverseChiSquaredDist() { + // Reference: scipy.special.gammaincc(v/2, v*sigma/(2*x)) for CDF + // InverseCDF: v*sigma / (2 * gammainccinv(v/2, p)) double true_mean = 0.2; - double true_median = 6.345811068141737d; + double true_median = 0.15758426609d; double true_pdf = 0.0000063457380298844403d; - double true_cdf = 0.50860033566176044d; + double true_cdf = 0.9999884277d; double true_icdf = 6.27d; var IX = new InverseChiSquared(7, (1d / 7d)); double pdf = IX.PDF(6.27d); @@ -73,7 +75,7 @@ public void Test_InverseChiSquaredDist() Assert.AreEqual(IX.Median, true_median, 0.0001d); Assert.AreEqual(IX.PDF(6.27d), true_pdf, 0.0001d); Assert.AreEqual(IX.CDF(6.27d), true_cdf, 0.0001d); - Assert.AreEqual(IX.InverseCDF(IX.CDF(6.27d)), true_icdf, 0.0001d); + Assert.AreEqual(IX.InverseCDF(IX.CDF(6.27d)), true_icdf, 0.001d); } /// @@ -139,11 +141,12 @@ public void Test_Mean() [TestMethod()] public void Test_Median() { + // Reference: v*sigma / (2 * gammainccinv(v/2, 0.5)) var IX = new InverseChiSquared(); - Assert.AreEqual(0.93418, IX.Median, 1e-04); + Assert.AreEqual(1.07046, IX.Median, 1e-04); var IX2 = new InverseChiSquared(7, 1); - Assert.AreEqual(0.906544, IX2.Median, 1e-04); + Assert.AreEqual(1.10309, IX2.Median, 1e-04); } /// @@ -231,8 +234,9 @@ public void Test_PDF() [TestMethod()] public void Test_CDF() { + // Reference: scipy.special.gammaincc(3.5, 3.5/5) = 0.985571264449 var IX = new InverseChiSquared(7,1); - Assert.AreEqual(1.1184e-05, IX.CDF(5), 1e-09); + Assert.AreEqual(0.985571264449d, IX.CDF(5), 1e-06); } /// @@ -244,7 +248,51 @@ public void Test_InverseCDF() var IX = new InverseChiSquared(); Assert.AreEqual(0, IX.InverseCDF(0)); Assert.AreEqual(double.PositiveInfinity,IX.InverseCDF(1)); - Assert.AreEqual(1.17807, IX.InverseCDF(0.3), 1e-04); + // Reference: scipy.stats.chi2.isf => v*sigma / chi2.isf(p, v) + Assert.AreEqual(0.84884, IX.InverseCDF(0.3), 1e-04); + } + + /// + /// Verify CDF and InverseCDF are correct and mutually inverse. + /// Reference: scipy.stats.chi2.sf(v*sigma/x, v) for CDF; v*sigma/chi2.isf(p, v) for InverseCDF. + /// + [TestMethod] + public void Test_CDF_InverseCDF_Roundtrip() + { + // v=10, sigma=1: CDF values from scipy.stats.chi2.sf(v*sigma/x, v) + var dist = new InverseChiSquared(10, 1); + Assert.AreEqual(0.0292526881, dist.CDF(0.5), 1E-6); + Assert.AreEqual(0.4404932851, dist.CDF(1.0), 1E-6); + Assert.AreEqual(0.8911780189, dist.CDF(2.0), 1E-6); + + // CDF must be non-decreasing + Assert.IsLessThan(dist.CDF(1.0), dist.CDF(0.5)); + Assert.IsLessThan(dist.CDF(2.0), dist.CDF(1.0)); + + // InverseCDF values from v*sigma / scipy.stats.chi2.isf(p, v) + Assert.AreEqual(0.6255012152, dist.InverseCDF(0.1), 1E-4); + Assert.AreEqual(0.8488443635, dist.InverseCDF(0.3), 1E-4); + Assert.AreEqual(1.0704554778, dist.InverseCDF(0.5), 1E-4); + Assert.AreEqual(1.3760423551, dist.InverseCDF(0.7), 1E-4); + Assert.AreEqual(2.0554215430, dist.InverseCDF(0.9), 1E-4); + + // Roundtrip: InverseCDF(CDF(x)) ≈ x + foreach (double x in new[] { 0.5, 1.0, 2.0 }) + { + Assert.AreEqual(x, dist.InverseCDF(dist.CDF(x)), 1E-4, $"Roundtrip failed for x={x}"); + } + + // v=5, sigma=2: CDF values from scipy.stats.chi2.sf(v*sigma/x, v) + var dist2 = new InverseChiSquared(5, 2); + Assert.AreEqual(0.0012497306, dist2.CDF(0.5), 1E-6); + Assert.AreEqual(0.4158801870, dist2.CDF(2.0), 1E-6); + Assert.AreEqual(0.8491450361, dist2.CDF(5.0), 1E-6); + + // Roundtrip for v=5, sigma=2 + foreach (double x in new[] { 0.5, 2.0, 5.0 }) + { + Assert.AreEqual(x, dist2.InverseCDF(dist2.CDF(x)), 1E-3, $"Roundtrip failed for x={x}"); + } } } } diff --git a/Test_Numerics/Distributions/Univariate/Test_LnNormal.cs b/Test_Numerics/Distributions/Univariate/Test_LnNormal.cs index 30177e06..3bf8b97b 100644 --- a/Test_Numerics/Distributions/Univariate/Test_LnNormal.cs +++ b/Test_Numerics/Distributions/Univariate/Test_LnNormal.cs @@ -167,7 +167,7 @@ public void Test_ConditionalExpectation() double CE = LN.ConditionalExpectedValue(alpha); double true_CE = Math.Exp(LN.Mu + 0.5 * LN.Sigma * LN.Sigma) / (1d - alpha) * (1 - Normal.StandardCDF(Normal.StandardZ(alpha) - LN.Sigma)); - Assert.AreEqual(CE, true_CE, 1E-4); + Assert.AreEqual(true_CE, CE, 1E-4); } @@ -210,10 +210,10 @@ public void Test_Moments() { var dist = new LnNormal(10, 5); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_LogPearsonTypeIII.cs b/Test_Numerics/Distributions/Univariate/Test_LogPearsonTypeIII.cs index fd84945e..9f659e5d 100644 --- a/Test_Numerics/Distributions/Univariate/Test_LogPearsonTypeIII.cs +++ b/Test_Numerics/Distributions/Univariate/Test_LogPearsonTypeIII.cs @@ -190,15 +190,15 @@ public void Test_LP3_StandardError() // Method of Moments var LP3 = new LogPearsonTypeIII(2.26878d, 0.10699d, -0.04061d); - double qVar999 = Math.Sqrt(LP3.QuantileVariance(0.99d, 69, ParameterEstimationMethod.MethodOfMoments)); - double true_qVar999 = 25.053d; - Assert.IsLessThan(0.01d, (qVar999 - true_qVar999) / true_qVar999); + double qVar99 = Math.Sqrt(LP3.QuantileVariance(0.99d, 69, ParameterEstimationMethod.MethodOfMoments)); + double true_qVar99 = 25.053d; + Assert.IsLessThan(0.01d, (qVar99 - true_qVar99) / true_qVar99); // Maximum Likelihood LP3 = new LogPearsonTypeIII(2.26878d, 0.10621d, -0.02925d); - qVar999 = Math.Sqrt(LP3.QuantileVariance(0.99d, 69, ParameterEstimationMethod.MaximumLikelihood)); - true_qVar999 = 25d; - Assert.IsLessThan(0.01d, (qVar999 - true_qVar999) / true_qVar999); + qVar99 = Math.Sqrt(LP3.QuantileVariance(0.99d, 69, ParameterEstimationMethod.MaximumLikelihood)); + true_qVar99 = 25d; + Assert.IsLessThan(0.01d, (qVar99 - true_qVar99) / true_qVar99 ); } @@ -286,13 +286,13 @@ public void Test_Mode() public void Test_Minimum() { var LP3 = new LogPearsonTypeIII(); - Assert.AreEqual(0, LP3.Minimum); + Assert.AreEqual(0, LP3.Minimum ); var LP3ii = new LogPearsonTypeIII(1,1,1); Assert.AreEqual(0.1, LP3ii.Minimum, 1e-05); var LP3iii = new LogPearsonTypeIII(1, -1, 1); - Assert.AreEqual(0, LP3iii.Minimum); + Assert.AreEqual(0,LP3iii.Minimum); } /// @@ -329,7 +329,7 @@ public void Test_PDF() public void Test_CDF() { var LP3 = new LogPearsonTypeIII(); - Assert.AreEqual(0, LP3.CDF(-1)); + Assert.AreEqual(0,LP3.CDF(-1)); Assert.AreEqual(9.8658e-10, LP3.CDF(1), 1e-13); } diff --git a/Test_Numerics/Distributions/Univariate/Test_Logistic.cs b/Test_Numerics/Distributions/Univariate/Test_Logistic.cs index 0fa93603..27de1b98 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Logistic.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Logistic.cs @@ -128,7 +128,7 @@ public void Test_Logistic_Quantile() Assert.IsLessThan(0.01d, (q100 - true_100) / true_100); double p = LO.CDF(q100); double true_p = 0.99d; - Assert.AreEqual(p, true_p); + Assert.AreEqual(true_p, p); } /// @@ -211,10 +211,10 @@ public void Test_Moments() { var dist = new Logistic(); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_Mixture.cs b/Test_Numerics/Distributions/Univariate/Test_Mixture.cs index dc80112d..493211a8 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Mixture.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Mixture.cs @@ -232,5 +232,52 @@ public void Test_3D_Mixture_MLE() } + /// + /// Test the inverse CDF of a zero-inflated mixture. + /// + [TestMethod] + public void Test_Mixture_InverseCDF_ZeroInflated() + { + // Test with 2 components and zero-inflation + var mix = new Mixture(new[] { 0.3, 0.6 }, new[] { new Normal(3, 0.1), new Normal(5, 2) }); + mix.IsZeroInflated = true; + mix.ZeroWeight = 0.1; + + // For probability <= ZeroWeight, InverseCDF should return 0 + Assert.AreEqual(0, mix.InverseCDF(0.05)); + Assert.AreEqual(0, mix.InverseCDF(0.1)); + + // For probability > ZeroWeight, verify InverseCDF inverts CDF + var xVals = Tools.Sequence(0.1d, 15d, 0.5d); + for (int i = 0; i < xVals.Length; i++) + { + double p = mix.CDF(xVals[i]); + if (p > mix.ZeroWeight && p < 1.0) + { + Assert.AreEqual(xVals[i], mix.InverseCDF(p), 1E-4); + } + } + + // Test with 1 component and zero-inflation + var mix1 = new Mixture(new[] { 0.7 }, new[] { new Normal(10, 2) }); + mix1.IsZeroInflated = true; + mix1.ZeroWeight = 0.3; + + // For probability <= ZeroWeight, InverseCDF should return 0 + Assert.AreEqual(0, mix1.InverseCDF(0.15)); + Assert.AreEqual(0, mix1.InverseCDF(0.3)); + + // For probability > ZeroWeight, verify InverseCDF inverts CDF + var xVals1 = Tools.Sequence(1d, 20d, 0.5d); + for (int i = 0; i < xVals1.Length; i++) + { + double p = mix1.CDF(xVals1[i]); + if (p > mix1.ZeroWeight && p < 1.0) + { + Assert.AreEqual(xVals1[i], mix1.InverseCDF(p), 1E-4); + } + } + } + } } diff --git a/Test_Numerics/Distributions/Univariate/Test_NoncentralT.cs b/Test_Numerics/Distributions/Univariate/Test_NoncentralT.cs index f8e17c82..c25eee84 100644 --- a/Test_Numerics/Distributions/Univariate/Test_NoncentralT.cs +++ b/Test_Numerics/Distributions/Univariate/Test_NoncentralT.cs @@ -114,12 +114,12 @@ public void Test_NoncentralT_InverseCDF() public void Test_Construction() { var t = new NoncentralT(); - Assert.AreEqual(10,t.DegreesOfFreedom); - Assert.AreEqual(0, t.Noncentrality); + Assert.AreEqual(10d, t.DegreesOfFreedom); + Assert.AreEqual(0d, t.Noncentrality); var t2 = new NoncentralT(1, 1); - Assert.AreEqual(1, t2.DegreesOfFreedom); - Assert.AreEqual(1, t2.Noncentrality); + Assert.AreEqual(1d, t2.DegreesOfFreedom); + Assert.AreEqual(1d, t2.Noncentrality); } /// @@ -270,5 +270,144 @@ public void Test_InverseCDF() Assert.AreEqual(double.PositiveInfinity, t.InverseCDF(1)); Assert.AreEqual(-0.26018, t.InverseCDF(0.4), 1e-04); } + + /// + /// Validates NoncentralT InverseCDF against Stedinger (1983) Table 1. + /// Table 1: Percentage points of ζ(0.90)-distribution for the 10-year event. + /// Relationship: ζ_α(p) = NCT.InverseCDF(α; df=n-1, δ=z_p·√n) / √n + /// + [TestMethod()] + public void Test_NoncentralT_StedsingerTable1() + { + double zp = Normal.StandardZ(0.90); + + // Exact values from Stedinger (1983) Table 1 (p=0.90) + // { sampleSize, alpha, expected zeta } + var table = new[,] + { + { 10d, 0.005, 0.436 }, { 10d, 0.050, 0.712 }, { 10d, 0.250, 1.043 }, + { 10d, 0.750, 1.671 }, { 10d, 0.950, 2.355 }, { 10d, 0.995, 3.368 }, + { 20d, 0.005, 0.651 }, { 20d, 0.050, 0.858 }, { 20d, 0.250, 1.104 }, + { 20d, 0.750, 1.528 }, { 20d, 0.950, 1.926 }, { 20d, 0.995, 2.423 }, + { 50d, 0.005, 0.857 }, { 50d, 0.050, 1.000 }, { 50d, 0.250, 1.164 }, + { 50d, 0.750, 1.426 }, { 50d, 0.950, 1.646 }, { 50d, 0.995, 1.890 }, + { 100d, 0.005, 0.970 }, { 100d, 0.050, 1.077 }, { 100d, 0.250, 1.196 }, + { 100d, 0.750, 1.380 }, { 100d, 0.950, 1.527 }, { 100d, 0.995, 1.683 }, + }; + + for (int i = 0; i < table.GetLength(0); i++) + { + int n = (int)table[i, 0]; + double alpha = table[i, 1]; + double expectedZeta = table[i, 2]; + + int df = n - 1; + double delta = zp * Math.Sqrt(n); + var nct = new NoncentralT(df, delta); + double computedZeta = nct.InverseCDF(alpha) / Math.Sqrt(n); + + Assert.AreEqual(expectedZeta, computedZeta, 0.002, + $"Table 1 mismatch: n={n}, α={alpha}, expected={expectedZeta}, computed={computedZeta:F4}"); + } + } + + /// + /// Validates NoncentralT InverseCDF against Stedinger (1983) Table 3. + /// Table 3: Percentage points of ζ(0.99)-distribution for the 100-year event. + /// + [TestMethod()] + public void Test_NoncentralT_StedsingerTable3() + { + double zp = Normal.StandardZ(0.99); + + // Exact values from Stedinger (1983) Table 3 (p=0.99) + var table = new[,] + { + { 10d, 0.005, 1.232 }, { 10d, 0.050, 1.562 }, { 10d, 0.250, 2.008 }, + { 10d, 0.750, 2.927 }, { 10d, 0.950, 3.981 }, { 10d, 0.995, 5.582 }, + { 20d, 0.005, 1.483 }, { 20d, 0.050, 1.749 }, { 20d, 0.250, 2.085 }, + { 20d, 0.750, 2.697 }, { 20d, 0.950, 3.295 }, { 20d, 0.995, 4.059 }, + { 50d, 0.005, 1.744 }, { 50d, 0.050, 1.936 }, { 50d, 0.250, 2.163 }, + { 50d, 0.750, 2.538 }, { 50d, 0.950, 2.862 }, { 50d, 0.995, 3.230 }, + { 100d, 0.005, 1.894 }, { 100d, 0.050, 2.040 }, { 100d, 0.250, 2.207 }, + { 100d, 0.750, 2.470 }, { 100d, 0.950, 2.684 }, { 100d, 0.995, 2.915 }, + }; + + for (int i = 0; i < table.GetLength(0); i++) + { + int n = (int)table[i, 0]; + double alpha = table[i, 1]; + double expectedZeta = table[i, 2]; + + int df = n - 1; + double delta = zp * Math.Sqrt(n); + var nct = new NoncentralT(df, delta); + double computedZeta = nct.InverseCDF(alpha) / Math.Sqrt(n); + + Assert.AreEqual(expectedZeta, computedZeta, 0.002, + $"Table 3 mismatch: n={n}, α={alpha}, expected={expectedZeta}, computed={computedZeta:F4}"); + } + } + + /// + /// Tests NoncentralT CDF and InverseCDF with large noncentrality parameters + /// encountered in real confidence interval calculations (δ > 20). + /// + [TestMethod()] + public void Test_NoncentralT_LargeNoncentrality() + { + double[] probs = { 0.01, 0.05, 0.25, 0.50, 0.75, 0.95, 0.99 }; + + // δ = z_0.99 * √100 ≈ 23.3 + var nct1 = new NoncentralT(99, Normal.StandardZ(0.99) * Math.Sqrt(100)); + foreach (double p in probs) + { + double x = nct1.InverseCDF(p); + Assert.IsFalse(double.IsNaN(x), $"InverseCDF({p}) returned NaN for δ≈23.3"); + Assert.IsFalse(double.IsInfinity(x), $"InverseCDF({p}) returned Infinity for δ≈23.3"); + double roundtrip = nct1.CDF(x); + Assert.AreEqual(p, roundtrip, 1e-4, $"CDF(InverseCDF({p})) roundtrip failed for δ≈23.3"); + } + + // δ = z_0.99 * √200 ≈ 32.9 + var nct2 = new NoncentralT(199, Normal.StandardZ(0.99) * Math.Sqrt(200)); + foreach (double p in probs) + { + double x = nct2.InverseCDF(p); + Assert.IsFalse(double.IsNaN(x), $"InverseCDF({p}) returned NaN for δ≈32.9"); + Assert.IsFalse(double.IsInfinity(x), $"InverseCDF({p}) returned Infinity for δ≈32.9"); + double roundtrip = nct2.CDF(x); + Assert.AreEqual(p, roundtrip, 1e-4, $"CDF(InverseCDF({p})) roundtrip failed for δ≈32.9"); + } + } + + /// + /// Verifies that NoncentralTConfidenceIntervals produces results equivalent to + /// MonteCarloConfidenceIntervals. The noncentral-t method is exact (Stedinger 1983); + /// the Monte Carlo method is approximate. + /// + [TestMethod()] + public void Test_NoncentralT_CI_vs_MonteCarlo_CI() + { + var dist = new Normal(100, 15); + int sampleSize = 30; + var quantiles = new double[] { 0.01, 0.10, 0.50, 0.90, 0.99 }; + var percentiles = new double[] { 0.05, 0.25, 0.50, 0.75, 0.95 }; + + var nctCI = dist.NoncentralTConfidenceIntervals(sampleSize, quantiles, percentiles); + var mcCI = dist.MonteCarloConfidenceIntervals(sampleSize, 100000, quantiles, percentiles); + + for (int i = 0; i < quantiles.Length; i++) + { + for (int j = 0; j < percentiles.Length; j++) + { + double nctVal = nctCI[i, j]; + double mcVal = mcCI[i, j]; + double tol = Math.Abs(nctVal) > 1.0 ? 0.02 * Math.Abs(nctVal) : 0.5; + Assert.AreEqual(nctVal, mcVal, tol, + $"NCT vs MC: quantile={quantiles[i]}, percentile={percentiles[j]}: NCT={nctVal:F4}, MC={mcVal:F4}"); + } + } + } } } diff --git a/Test_Numerics/Distributions/Univariate/Test_Normal.cs b/Test_Numerics/Distributions/Univariate/Test_Normal.cs index 889f4e54..a231623b 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Normal.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Normal.cs @@ -101,8 +101,8 @@ public void Test_Normal_LMOM_Fit() double u2 = norm.Sigma; double true_u1 = 9.957516d; double true_u2 = 3.513431d; - Assert.AreEqual(u1, true_u1, 0.0001d); - Assert.AreEqual(u2, true_u2, 0.0001d); + Assert.AreEqual(true_u1, u1, 0.0001d); + Assert.AreEqual(true_u2, u2, 0.0001d); var lmom = norm.LinearMomentsFromParameters(norm.GetParameters); Assert.AreEqual(9.9575163d, lmom[0], 0.0001d); Assert.AreEqual(1.9822411d, lmom[1], 0.0001d); @@ -232,10 +232,10 @@ public void Test_Moments() { var dist = new Normal(5, 9); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_Pareto.cs b/Test_Numerics/Distributions/Univariate/Test_Pareto.cs index b160758a..86de42ba 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Pareto.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Pareto.cs @@ -72,20 +72,20 @@ public void Test_ParetoDist() double true_icdf05 = 0.4272d; double true_icdf95 = 1.1401d; var PA = new Pareto(0.42d, 3d); - Assert.AreEqual(PA.Mean, true_mean, 0.0001d); - Assert.AreEqual(PA.Median, true_median, 0.0001d); - Assert.AreEqual(PA.Mode, true_mode, 0.0001d); - Assert.AreEqual(PA.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(PA.PDF(1.4d), true_pdf, 0.0001d); - Assert.AreEqual(PA.CDF(1.4d), true_cdf, 0.0001d); - Assert.AreEqual(PA.InverseCDF(PA.CDF(1.4d)), true_icdf, 0.0001d); - Assert.AreEqual(PA.InverseCDF(0.05d), true_icdf05, 0.0001d); - Assert.AreEqual(PA.InverseCDF(0.95d), true_icdf95, 0.0001d); + Assert.AreEqual(true_mean, PA.Mean, 0.0001d); + Assert.AreEqual(true_median, PA.Median, 0.0001d); + Assert.AreEqual(true_mode, PA.Mode, 0.0001d); + Assert.AreEqual(true_stdDev, PA.StandardDeviation, 0.0001d); + Assert.AreEqual(true_pdf, PA.PDF(1.4d), 0.0001d); + Assert.AreEqual(true_cdf, PA.CDF(1.4d), 0.0001d); + Assert.AreEqual(true_icdf, PA.InverseCDF(PA.CDF(1.4d)), 0.0001d); + Assert.AreEqual(true_icdf05, PA.InverseCDF(0.05d), 0.0001d); + Assert.AreEqual(true_icdf95, PA.InverseCDF(0.95d), 0.0001d); PA.SetParameters(new[] { 1d, 10d }); double true_skew = 2.8111d; double true_kurt = 17.8286d; - Assert.AreEqual(PA.Skewness, true_skew, 0.0001d); - Assert.AreEqual(PA.Kurtosis, true_kurt, 0.0001d); + Assert.AreEqual(true_skew, PA.Skewness, 0.0001d); + Assert.AreEqual(true_kurt, PA.Kurtosis, 0.0001d); } /// @@ -140,10 +140,10 @@ public void Test_Moments() { var dist = new Pareto(); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_PearsonTypeIII.cs b/Test_Numerics/Distributions/Univariate/Test_PearsonTypeIII.cs index 45606a90..f18a15c2 100644 --- a/Test_Numerics/Distributions/Univariate/Test_PearsonTypeIII.cs +++ b/Test_Numerics/Distributions/Univariate/Test_PearsonTypeIII.cs @@ -120,14 +120,14 @@ public void Test_P3_LMOM_Fit() double true_x = 863.4104d; double true_a = 10.02196d; double true_b = 78.36751d; - Assert.AreEqual(x, true_x, 0.001d); - Assert.AreEqual(a, true_a, 0.001d); - Assert.AreEqual(b, true_b, 0.001d); + Assert.AreEqual(true_x, x, 0.001d); + Assert.AreEqual(true_a, a, 0.001d); + Assert.AreEqual(true_b, b, 0.001d); var lmom = P3.LinearMomentsFromParameters(P3.GetParameters); Assert.AreEqual(1648.806d, lmom[0], 0.001d); - Assert.AreEqual(138.2366d, lmom[1], 0.001d); - Assert.AreEqual(0.1033889d, lmom[2], 0.001d); - Assert.AreEqual(0.1258521d, lmom[3], 0.001d); + Assert.AreEqual(138.2366d, lmom[1], 0.001d); + Assert.AreEqual(0.1033889d, lmom[2], 0.001d); + Assert.AreEqual(0.1258521d, lmom[3], 0.001d); } /// @@ -212,15 +212,15 @@ public void Test_P3_StandardError() // Method of Moments var P3 = new PearsonTypeIII(191.31739d, 47.96161d, 0.86055d); - double qVar999 = Math.Sqrt(P3.QuantileVariance(0.99d, 69, ParameterEstimationMethod.MethodOfMoments)); - double true_qVar999 = 27.175d; - Assert.IsLessThan(0.01d, (qVar999 - true_qVar999) / true_qVar999); + double qVar99 = Math.Sqrt(P3.QuantileVariance(0.99d, 69, ParameterEstimationMethod.MethodOfMoments)); + double true_qVar99 = 27.175d; + Assert.IsLessThan(0.01d, (qVar99 - true_qVar99) / true_qVar99); // Maximum Likelihood P3 = new PearsonTypeIII(191.31739d, 47.01925d, 0.61897d); - qVar999 = Math.Sqrt(P3.QuantileVariance(0.99d, 69, ParameterEstimationMethod.MaximumLikelihood)); - true_qVar999 = 20.045d; - Assert.IsLessThan(0.01d, (qVar999 - true_qVar999) / true_qVar999); + qVar99 = Math.Sqrt(P3.QuantileVariance(0.99d, 69, ParameterEstimationMethod.MaximumLikelihood)); + true_qVar99 = 20.045d; + Assert.IsLessThan(0.01d, (qVar99 - true_qVar99) / true_qVar99 ); } /// @@ -279,10 +279,10 @@ public void Test_Moments() { var dist = new PearsonTypeIII(); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_Pert.cs b/Test_Numerics/Distributions/Univariate/Test_Pert.cs index bbc4ea78..c9f1d8d4 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Pert.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Pert.cs @@ -70,12 +70,12 @@ public void Test_PertDist() double true_cdf = GB.CDF(1.27); double true_icdf = 1.27d; - Assert.AreEqual(P.Mean, true_mean, 0.0001d); - Assert.AreEqual(P.Median, true_median, 0.0001d); - Assert.AreEqual(P.Mode, true_mode, 0.0001d); - Assert.AreEqual(P.PDF(1.27d), true_pdf, 0.0001d); - Assert.AreEqual(P.CDF(1.27d), true_cdf, 0.0001d); - Assert.AreEqual(P.InverseCDF(P.CDF(1.27d)), true_icdf, 0.0001d); + Assert.AreEqual(true_mean, P.Mean, 0.0001d); + Assert.AreEqual(true_median, P.Median, 0.0001d); + Assert.AreEqual(true_mode, P.Mode, 0.0001d); + Assert.AreEqual(true_pdf, P.PDF(1.27d), 0.0001d); + Assert.AreEqual(true_cdf, P.CDF(1.27d), 0.0001d); + Assert.AreEqual(true_icdf, P.InverseCDF(P.CDF(1.27d)), 0.0001d); } @@ -199,10 +199,10 @@ public void Test_Moments() { var dist = new Pert(-2, 10, 35); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_Poisson.cs b/Test_Numerics/Distributions/Univariate/Test_Poisson.cs index 0f7694b3..1d9c1fd1 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Poisson.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Poisson.cs @@ -73,16 +73,16 @@ public void Test_PoissonDist() double true_icdf05 = 1.0d; double true_icdf95 = 8.0d; var P = new Poisson(4.2d); - Assert.AreEqual(P.Mean, true_mean, 0.0001d); - Assert.AreEqual(P.Median, true_median, 0.0001d); - Assert.AreEqual(P.Mode, true_mode, 0.0001d); - Assert.AreEqual(P.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(P.Skewness, true_skew, 0.0001d); - Assert.AreEqual(P.Kurtosis, true_kurt, 0.0001d); - Assert.AreEqual(P.PDF(4.0d), true_pdf, 0.0001d); - Assert.AreEqual(P.CDF(4.0d), true_cdf, 0.0001d); - Assert.AreEqual(P.InverseCDF(0.05d), true_icdf05, 0.0001d); - Assert.AreEqual(P.InverseCDF(0.95d), true_icdf95, 0.0001d); + Assert.AreEqual(true_mean, P.Mean, 0.0001d); + Assert.AreEqual(true_median, P.Median, 0.0001d); + Assert.AreEqual(true_mode, P.Mode, 0.0001d); + Assert.AreEqual(true_stdDev, P.StandardDeviation, 0.0001d); + Assert.AreEqual(true_skew, P.Skewness, 0.0001d); + Assert.AreEqual(true_kurt, P.Kurtosis, 0.0001d); + Assert.AreEqual(true_pdf, P.PDF(4.0d), 0.0001d); + Assert.AreEqual(true_cdf, P.CDF(4.0d), 0.0001d); + Assert.AreEqual(true_icdf05, P.InverseCDF(0.05d), 0.0001d); + Assert.AreEqual(true_icdf95, P.InverseCDF(0.95d), 0.0001d); } /// @@ -133,10 +133,10 @@ public void Test_Moments() { var dist = new Poisson(10); var mom = dist.CentralMoments(1000); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_Rayleigh.cs b/Test_Numerics/Distributions/Univariate/Test_Rayleigh.cs index 66b5c9c6..62837e4f 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Rayleigh.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Rayleigh.cs @@ -69,12 +69,12 @@ public void Test_RayleighDist() double true_cdf = 0.99613407986052716d; double true_icdf = 1.4d; var R = new Rayleigh(0.42); - Assert.AreEqual(R.Mean, true_mean, 0.0001d); - Assert.AreEqual(R.Median, true_median, 0.0001d); - Assert.AreEqual(R.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(R.PDF(1.4d), true_pdf, 0.0001d); - Assert.AreEqual(R.CDF(1.4d), true_cdf, 0.0001d); - Assert.AreEqual(R.InverseCDF(true_cdf), true_icdf, 0.0001d); + Assert.AreEqual(true_mean, R.Mean, 0.0001d); + Assert.AreEqual(true_median, R.Median, 0.0001d); + Assert.AreEqual(true_stdDev, R.StandardDeviation, 0.0001d); + Assert.AreEqual(true_pdf, R.PDF(1.4d), 0.0001d); + Assert.AreEqual(true_cdf, R.CDF(1.4d), 0.0001d); + Assert.AreEqual(true_icdf, R.InverseCDF(true_cdf), 0.0001d); } /// @@ -84,7 +84,7 @@ public void Test_RayleighDist() public void Test_Construction() { var R = new Rayleigh(); - Assert.AreEqual(10,R.Sigma); + Assert.AreEqual(10, R.Sigma); var R2 = new Rayleigh(2); Assert.AreEqual(2, R2.Sigma); @@ -138,10 +138,10 @@ public void Test_Moments() public void Test_Mean() { var R = new Rayleigh(); - Assert.AreEqual(12.53314, R.Mean, 1e-04); + Assert.AreEqual(12.53314, R.Mean, 1e-04); var R2 = new Rayleigh(1); - Assert.AreEqual(1.25331, R2.Mean, 1e-04); + Assert.AreEqual(1.25331, R2.Mean, 1e-04); } /// @@ -151,7 +151,7 @@ public void Test_Mean() public void Test_Median() { var R = new Rayleigh(); - Assert.AreEqual(11.7741, R.Median, 1e-04); + Assert.AreEqual(11.7741, R.Median, 1e-04); var R2 = new Rayleigh(1); Assert.AreEqual(1.1774, R2.Median, 1e-04); @@ -177,7 +177,7 @@ public void Test_Mode() public void Test_StandardDeviation() { var R = new Rayleigh(); - Assert.AreEqual(6.55136, R.StandardDeviation, 1e-05); + Assert.AreEqual(6.55136, R.StandardDeviation, 1e-05); var R2 = new Rayleigh(1); Assert.AreEqual(0.65513, R2.StandardDeviation, 1e-04); @@ -190,10 +190,10 @@ public void Test_StandardDeviation() public void Test_Skewness() { var R = new Rayleigh(); - Assert.AreEqual(0.63111, R.Skewness, 1e-04); + Assert.AreEqual(0.63111, R.Skewness, 1e-04); var R2 = new Rayleigh(1); - Assert.AreEqual(0.63111, R2.Skewness, 1e-04); + Assert.AreEqual(0.63111, R2.Skewness, 1e-04); } /// @@ -206,7 +206,7 @@ public void Test_Kurtosis() Assert.AreEqual(3.24508, R.Kurtosis, 1e-05); var R2 = new Rayleigh(1); - Assert.AreEqual(3.24508, R2.Kurtosis,1e-05); + Assert.AreEqual(3.24508, R2.Kurtosis, 1e-05); } /// @@ -216,8 +216,8 @@ public void Test_Kurtosis() public void Test_MinMax() { var R = new Rayleigh(); - Assert.AreEqual(0,R.Minimum); - Assert.AreEqual(double.PositiveInfinity,R.Maximum); + Assert.AreEqual(0, R.Minimum); + Assert.AreEqual(double.PositiveInfinity, R.Maximum); } /// @@ -227,11 +227,11 @@ public void Test_MinMax() public void Test_PDF() { var R = new Rayleigh(); - Assert.AreEqual(0,R.PDF(-1)); - Assert.AreEqual(9.9501e-03, R.PDF(1), 1e-06); + Assert.AreEqual(0, R.PDF(-1)); + Assert.AreEqual(9.9501e-03, R.PDF(1), 1e-06); var R2 = new Rayleigh(1); - Assert.AreEqual(0.019603, R.PDF(2), 1e-05); + Assert.AreEqual(0.019603, R.PDF(2), 1e-05); } /// @@ -242,7 +242,7 @@ public void Test_CDF() { var R = new Rayleigh(); Assert.AreEqual(0, R.CDF(-1)); - Assert.AreEqual(4.9875e-03, R.CDF(1),1e-04); + Assert.AreEqual(4.9875e-03, R.CDF(1), 1e-04); } /// @@ -253,8 +253,26 @@ public void Test_InverseCDF() { var R = new Rayleigh(); Assert.AreEqual(0, R.InverseCDF(0)); - Assert.AreEqual(double.PositiveInfinity,R.InverseCDF(1) ); - Assert.AreEqual(10.1076, R.InverseCDF(0.4), 1e-04); + Assert.AreEqual(double.PositiveInfinity, R.InverseCDF(1)); + Assert.AreEqual(10.1076, R.InverseCDF(0.4), 1e-04); + } + + /// + /// Verify Rayleigh MoM estimation recovers correct sigma from sample mean. + /// Reference: scipy.stats.rayleigh(scale=sigma).mean() = sigma * sqrt(pi/2). + /// Therefore sigma = mean / sqrt(pi/2), NOT sigma = stddev. + /// + [TestMethod()] + public void Test_MoM_Estimation() + { + // Generate a Rayleigh sample with known sigma=3.0 using a fixed seed + var sigma = 3.0; + var R = new Rayleigh(sigma); + // rayleigh(scale=3.0).mean() = 3.7599424119 + Assert.AreEqual(3.7599424119, R.Mean, 1E-4); + // rayleigh(scale=3.0).std() = 1.9652... + // MoM: sigma = mean / sqrt(pi/2) = 3.0 + Assert.AreEqual(sigma, R.Mean / Math.Sqrt(Math.PI / 2.0), 1E-10); } } } diff --git a/Test_Numerics/Distributions/Univariate/Test_StudentT.cs b/Test_Numerics/Distributions/Univariate/Test_StudentT.cs index e8e66234..c2637962 100644 --- a/Test_Numerics/Distributions/Univariate/Test_StudentT.cs +++ b/Test_Numerics/Distributions/Univariate/Test_StudentT.cs @@ -60,14 +60,14 @@ public class Test_StudentT [TestMethod()] public void Test_StudentT_PDF() { - var t = new StudentT(4.2d); + var t = new StudentT(4d); double pdf = t.PDF(1.4d); double result = 0.138377537135553d; - Assert.AreEqual(pdf, result, 1E-10); - t = new StudentT(2.5d, 0.5d, 4.2d); + Assert.AreEqual(result, pdf, 1E-10); + t = new StudentT(2.5d, 0.5d, 4d); pdf = t.PDF(1.4d); result = 0.0516476521260042d; - Assert.AreEqual(pdf, result, 1E-10); + Assert.AreEqual(result, pdf, 1E-10); } /// @@ -76,14 +76,14 @@ public void Test_StudentT_PDF() [TestMethod()] public void Test_StudentT_CDF() { - var t = new StudentT(4.2d); + var t = new StudentT(4d); double cdf = t.CDF(1.4d); double result = 0.882949686336585d; - Assert.AreEqual(cdf, result, 1E-10); - t = new StudentT(2.5d, 0.5d, 4.2d); + Assert.AreEqual(result, cdf, 1E-10); + t = new StudentT(2.5d, 0.5d, 4d); cdf = t.CDF(1.4d); result = 0.0463263350898173d; - Assert.AreEqual(cdf, result, 1E-10); + Assert.AreEqual(result, cdf, 1E-10); } /// @@ -92,16 +92,16 @@ public void Test_StudentT_CDF() [TestMethod()] public void Test_StudentT_InverseCDF() { - var t = new StudentT(4.2d); + var t = new StudentT(4d); double cdf = t.CDF(1.4d); double invcdf = t.InverseCDF(cdf); double result = 1.4d; - Assert.AreEqual(invcdf, result, 1E-2); - t = new StudentT(2.5d, 0.5d, 4.2d); + Assert.AreEqual(result, invcdf, 1E-2); + t = new StudentT(2.5d, 0.5d, 4d); cdf = t.CDF(1.4d); invcdf = t.InverseCDF(cdf); result = 1.4d; - Assert.AreEqual(invcdf, result, 1E-2); + Assert.AreEqual(result, invcdf, 1E-2); } /// @@ -111,14 +111,14 @@ public void Test_StudentT_InverseCDF() public void Test_Construction() { var t = new StudentT(); - Assert.AreEqual(0, t.Mu); - Assert.AreEqual(1, t.Sigma); - Assert.AreEqual(10, t.DegreesOfFreedom); + Assert.AreEqual(0d, t.Mu); + Assert.AreEqual(1d, t.Sigma); + Assert.AreEqual(10d, t.DegreesOfFreedom); var t2 = new StudentT(10, 10, 10); - Assert.AreEqual(10, t2.Mu); - Assert.AreEqual(10, t2.Sigma); - Assert.AreEqual(10, t2.DegreesOfFreedom); + Assert.AreEqual(10d, t2.Mu); + Assert.AreEqual(10d, t2.Sigma); + Assert.AreEqual(10d, t2.DegreesOfFreedom); } /// @@ -148,7 +148,7 @@ public void Test_ParametersToString() Assert.AreEqual("Scale (σ)", t.ParametersToString[1, 0]); Assert.AreEqual("Degrees of Freedom (ν)", t.ParametersToString[2, 0]); Assert.AreEqual("0", t.ParametersToString[0, 1]); - Assert.AreEqual("1", t.ParametersToString[1, 1]); + Assert.AreEqual("1", t.ParametersToString[1,1]); Assert.AreEqual("10", t.ParametersToString[2, 1]); } @@ -160,10 +160,10 @@ public void Test_Moments() { var dist = new StudentT(10, 1, 100); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -231,7 +231,7 @@ public void Test_Skewness() Assert.AreEqual(0, t.Skewness); var t2 = new StudentT(1, 1, 1); - Assert.AreEqual(double.NaN,t2.Skewness ); + Assert.AreEqual(double.NaN, t2.Skewness); } /// @@ -241,7 +241,7 @@ public void Test_Skewness() public void Test_Kurtosis() { var t = new StudentT(); - Assert.AreEqual(4, t.Kurtosis ); + Assert.AreEqual(4, t.Kurtosis); var t2 = new StudentT(1, 1, 4); Assert.AreEqual(double.PositiveInfinity, t2.Kurtosis); diff --git a/Test_Numerics/Distributions/Univariate/Test_Triangular.cs b/Test_Numerics/Distributions/Univariate/Test_Triangular.cs index 49756414..49f0a040 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Triangular.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Triangular.cs @@ -71,13 +71,13 @@ public void Test_TriangularDist() double true_icdf = 2.0d; var T = new Triangular(1, 3, 6); - Assert.AreEqual(T.Mean, true_mean, 0.0001d); - Assert.AreEqual(T.Median, true_median, 0.0001d); - Assert.AreEqual(T.Mode, true_mode, 0.0001d); - Assert.AreEqual(T.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(T.PDF(2.0d), true_pdf, 0.0001d); - Assert.AreEqual(T.CDF(2.0d), true_cdf, 0.0001d); - Assert.AreEqual(T.InverseCDF(true_cdf), true_icdf, 0.0001d); + Assert.AreEqual(true_mean, T.Mean, 0.0001d); + Assert.AreEqual(true_median, T.Median, 0.0001d); + Assert.AreEqual(true_mode, T.Mode, 0.0001d); + Assert.AreEqual(true_stdDev, T.StandardDeviation, 0.0001d); + Assert.AreEqual(true_pdf, T.PDF(2.0d), 0.0001d); + Assert.AreEqual(true_cdf, T.CDF(2.0d), 0.0001d); + Assert.AreEqual(true_icdf, T.InverseCDF(true_cdf), 0.0001d); } /// @@ -136,7 +136,7 @@ public void Test_Triangular_MLE() public void Test_Construction() { var T = new Triangular(); - Assert.AreEqual(0,T.Min); + Assert.AreEqual(0, T.Min); Assert.AreEqual(0.5, T.Mode); Assert.AreEqual(1, T.Max); @@ -172,9 +172,9 @@ public void Test_InvalidParameters() public void Test_ParametersToString() { var T = new Triangular(); - Assert.AreEqual("Min (a)",T.ParametersToString[0, 0] ); - Assert.AreEqual("Most Likely (c)",T.ParametersToString[1, 0] ); - Assert.AreEqual("Max (b)",T.ParametersToString[2, 0] ); + Assert.AreEqual("Min (a)", T.ParametersToString[0, 0]); + Assert.AreEqual("Most Likely (c)", T.ParametersToString[1, 0]); + Assert.AreEqual("Max (b)", T.ParametersToString[2, 0]); Assert.AreEqual("0", T.ParametersToString[0, 1]); Assert.AreEqual("0.5", T.ParametersToString[1, 1]); Assert.AreEqual("1", T.ParametersToString[2, 1]); @@ -188,10 +188,10 @@ public void Test_Moments() { var dist = new Triangular(1, 3, 6); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -204,7 +204,7 @@ public void Test_Mean() Assert.AreEqual(0.5, T.Mean); var T2 = new Triangular(1, 3, 6); - Assert.AreEqual(3.3333, T2.Mean, 1e-04); + Assert.AreEqual(3.3333, T2.Mean, 1e-04); } /// @@ -217,7 +217,7 @@ public void Test_Median() Assert.AreEqual(0.5, T.Median); var T2 = new Triangular(1,3,6); - Assert.AreEqual(3.26138, T2.Median, 1e-05); + Assert.AreEqual(3.26138, T2.Median, 1e-05); } /// @@ -243,7 +243,7 @@ public void Test_StandardDeviation() Assert.AreEqual(0.20412, T.StandardDeviation, 1e-04); var T2 = new Triangular(1, 3, 6); - Assert.AreEqual(1.02739, T2.StandardDeviation, 1e-04); + Assert.AreEqual(1.02739, T2.StandardDeviation, 1e-04); } /// @@ -291,13 +291,13 @@ public void Test_MinMax() public void Test_PDF() { var T = new Triangular(); - Assert.AreEqual(0,T.PDF(-1)); + Assert.AreEqual(0, T.PDF(-1)); Assert.AreEqual(1.6, T.PDF(0.4)); Assert.AreEqual(1.6, T.PDF(0.6)); Assert.AreEqual(2, T.PDF(0.5)); var T2 = new Triangular(1, 3, 6); - Assert.AreEqual(0.2, T2.PDF(2), 1e-04); + Assert.AreEqual(0.2, T2.PDF(2), 1e-04); } /// @@ -313,7 +313,7 @@ public void Test_CDF() Assert.AreEqual(0.68, T.CDF(0.6), 1e-04); var T2 = new Triangular(1,3, 6); - Assert.AreEqual(0.1, T2.CDF(2), 1e-04); + Assert.AreEqual(0.1, T2.CDF(2), 1e-04); } /// @@ -324,8 +324,8 @@ public void Test_InverseCDF() { var T = new Triangular(); Assert.AreEqual(0, T.InverseCDF(0)); - Assert.AreEqual(1,T.InverseCDF(1)); - Assert.AreEqual(0.31622, T.InverseCDF(0.2), 1e-04); + Assert.AreEqual(1, T.InverseCDF(1)); + Assert.AreEqual(0.31622, T.InverseCDF(0.2), 1e-04); Assert.AreEqual(0.5, T.InverseCDF(0.5)); } } diff --git a/Test_Numerics/Distributions/Univariate/Test_TruncatedDistribution.cs b/Test_Numerics/Distributions/Univariate/Test_TruncatedDistribution.cs index 9af0500e..ffd2a5e4 100644 --- a/Test_Numerics/Distributions/Univariate/Test_TruncatedDistribution.cs +++ b/Test_Numerics/Distributions/Univariate/Test_TruncatedDistribution.cs @@ -59,27 +59,27 @@ public void Test_TruncatedNormalDist() var p = tn.CDF(1.5); var q = tn.InverseCDF(p); - Assert.AreEqual(0.9786791, d, 1E-5); - Assert.AreEqual(0.3460251, p, 1E-5); - Assert.AreEqual(1.5, q, 1E-5); + Assert.AreEqual(0.9786791, d, 1E-5); + Assert.AreEqual(0.3460251, p, 1E-5); + Assert.AreEqual(1.5, q, 1E-5); tn = new TruncatedDistribution(new Normal(10, 3), 8, 25); d = tn.PDF(12.75); p = tn.CDF(12.75); q = tn.InverseCDF(p); - Assert.AreEqual(0.1168717, d, 1E-5); - Assert.AreEqual(0.7596566, p, 1E-5); - Assert.AreEqual(12.75, q, 1E-5); + Assert.AreEqual(0.1168717, d, 1E-5); + Assert.AreEqual(0.7596566, p, 1E-5); + Assert.AreEqual(12.75, q, 1E-5); tn = new TruncatedDistribution(new Normal(0, 3), 0, 9); d = tn.PDF(4.5); p = tn.CDF(4.5); q = tn.InverseCDF(p); - Assert.AreEqual(0.08657881, d, 1E-5); - Assert.AreEqual(0.868731, p, 1E-5); - Assert.AreEqual(4.5, q, 1E-5); + Assert.AreEqual(0.08657881, d, 1E-5); + Assert.AreEqual(0.868731, p, 1E-5); + Assert.AreEqual(4.5, q, 1E-5); } @@ -90,7 +90,7 @@ public void Test_TruncatedNormalDist() public void Test_Construction() { var tn = new TruncatedDistribution(new Normal(0.5, 0.2), 0, 1); - Assert.AreEqual(0.5,((Normal)tn.BaseDistribution).Mu); + Assert.AreEqual(0.5, ((Normal)tn.BaseDistribution).Mu); Assert.AreEqual(0.2, ((Normal)tn.BaseDistribution).Sigma); Assert.AreEqual(0, tn.Min); Assert.AreEqual(1, tn.Max); @@ -109,7 +109,7 @@ public void Test_Construction() public void Test_ParametersToString() { var tn = new TruncatedDistribution(new Normal(0.5, 0.2), 0, 1); - Assert.AreEqual("Mean (µ)",tn.ParametersToString[0, 0]); + Assert.AreEqual("Mean (µ)", tn.ParametersToString[0, 0]); Assert.AreEqual("Std Dev (σ)", tn.ParametersToString[1, 0]); Assert.AreEqual("Min", tn.ParametersToString[2, 0]); Assert.AreEqual("Max", tn.ParametersToString[3, 0]); @@ -150,7 +150,7 @@ public void Test_Mean() public void Test_Median() { var tn = new TruncatedDistribution(new Normal(0.5, 0.2), 0, 1); - Assert.AreEqual(0.5, tn.Median, 1e-4); + Assert.AreEqual(0.5, tn.Median, 1e-4); } /// @@ -170,7 +170,7 @@ public void Test_Mode() public void Test_StandardDeviation() { var tn = new TruncatedDistribution(new Normal(0.5, 0.2), 0, 1); - Assert.AreEqual(0.19091, tn.StandardDeviation, 1e-4); + Assert.AreEqual(0.19091, tn.StandardDeviation, 1e-4); } /// @@ -180,7 +180,7 @@ public void Test_StandardDeviation() public void Test_Skewness() { var tn = new TruncatedDistribution(new Normal(0.5, 0.2), 0, 1); - Assert.AreEqual(0, tn.Skewness, 1E-4); + Assert.AreEqual(0, tn.Skewness, 1E-4); } /// @@ -190,7 +190,7 @@ public void Test_Skewness() public void Test_Kurtosis() { var tn = new TruncatedDistribution(new Normal(0.5, 0.2), 0, 1); - Assert.AreEqual(2.62422, tn.Kurtosis, 1e-04); + Assert.AreEqual(2.62422, tn.Kurtosis, 1e-04); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_TruncatedNormal.cs b/Test_Numerics/Distributions/Univariate/Test_TruncatedNormal.cs index 92d1a714..051a2e5a 100644 --- a/Test_Numerics/Distributions/Univariate/Test_TruncatedNormal.cs +++ b/Test_Numerics/Distributions/Univariate/Test_TruncatedNormal.cs @@ -66,16 +66,16 @@ public void Test_TruncatedNormalDist() var p = tn.CDF(1.5); var q = tn.InverseCDF(p); - Assert.AreEqual(0.9786791, d, 1E-5); - Assert.AreEqual(0.3460251, p, 1E-5); - Assert.AreEqual(1.5, q, 1E-5); + Assert.AreEqual(0.9786791, d, 1E-5); + Assert.AreEqual(0.3460251, p, 1E-5); + Assert.AreEqual(1.5, q, 1E-5); tn = new TruncatedNormal(10, 3, 8, 25); d = tn.PDF(12.75); p = tn.CDF(12.75); q = tn.InverseCDF(p); - Assert.AreEqual(0.1168717, d ,1E-5); + Assert.AreEqual(0.1168717, d, 1E-5); Assert.AreEqual(0.7596566, p, 1E-5); Assert.AreEqual(12.75, q, 1E-5); @@ -97,16 +97,16 @@ public void Test_TruncatedNormalDist() public void Test_Construction() { var tn = new TruncatedNormal(); - Assert.AreEqual(0.5,tn.Mu ); - Assert.AreEqual(0.2,tn.Sigma ); - Assert.AreEqual(0,tn.Min); - Assert.AreEqual(1,tn.Max); + Assert.AreEqual(0.5, tn.Mu); + Assert.AreEqual(0.2, tn.Sigma); + Assert.AreEqual(0, tn.Min); + Assert.AreEqual(1, tn.Max); var tn2 = new TruncatedNormal(1, 1, 1, 2); - Assert.AreEqual(1,tn2.Mu); - Assert.AreEqual(1,tn2.Sigma); - Assert.AreEqual(1,tn2.Min); - Assert.AreEqual(2,tn2.Max); + Assert.AreEqual(1, tn2.Mu); + Assert.AreEqual(1, tn2.Sigma); + Assert.AreEqual(1, tn2.Min); + Assert.AreEqual(2, tn2.Max); } /// @@ -135,11 +135,11 @@ public void Test_InvalidParameters() public void Test_ParametersToString() { var tn = new TruncatedNormal(); - Assert.AreEqual("Mean (µ)",tn.ParametersToString[0, 0]); - Assert.AreEqual("Std Dev (σ)",tn.ParametersToString[1, 0]); - Assert.AreEqual("Min",tn.ParametersToString[2, 0] ); - Assert.AreEqual("Max", tn.ParametersToString[3, 0] ); - Assert.AreEqual("0.5",tn.ParametersToString[0, 1]); + Assert.AreEqual("Mean (µ)", tn.ParametersToString[0, 0]); + Assert.AreEqual("Std Dev (σ)", tn.ParametersToString[1, 0]); + Assert.AreEqual("Min", tn.ParametersToString[2, 0]); + Assert.AreEqual("Max", tn.ParametersToString[3, 0]); + Assert.AreEqual("0.5", tn.ParametersToString[0, 1]); Assert.AreEqual("0.2", tn.ParametersToString[1, 1]); Assert.AreEqual("0", tn.ParametersToString[2, 1]); Assert.AreEqual("1", tn.ParametersToString[3, 1]); @@ -166,7 +166,7 @@ public void Test_Moments() public void Test_Mean() { var tn = new TruncatedNormal(); - Assert.AreEqual(0.5,tn.Mean); + Assert.AreEqual(0.5, tn.Mean); } /// @@ -216,7 +216,7 @@ public void Test_Skewness() public void Test_Kurtosis() { var tn = new TruncatedNormal(); - Assert.AreEqual(2.62422, tn.Kurtosis, 1e-04); + Assert.AreEqual(2.62422, tn.Kurtosis, 1e-04); } /// diff --git a/Test_Numerics/Distributions/Univariate/Test_Uniform.cs b/Test_Numerics/Distributions/Univariate/Test_Uniform.cs index 56f17684..6a60657e 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Uniform.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Uniform.cs @@ -69,12 +69,12 @@ public void Test_UniformDist() double true_cdf = 0.70588235294117641d; double true_icdf = 0.9d; var U = new Uniform(0.42d, 1.1d); - Assert.AreEqual(U.Mean, true_mean, 0.0001d); - Assert.AreEqual(U.Median, true_median, 0.0001d); - Assert.AreEqual(U.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(U.PDF(0.9d), true_pdf, 0.0001d); - Assert.AreEqual(U.CDF(0.9d), true_cdf, 0.0001d); - Assert.AreEqual(U.InverseCDF(true_cdf), true_icdf, 0.0001d); + Assert.AreEqual(true_mean, U.Mean, 0.0001d); + Assert.AreEqual(true_median, U.Median, 0.0001d); + Assert.AreEqual(true_stdDev, U.StandardDeviation, 0.0001d); + Assert.AreEqual(true_pdf, U.PDF(0.9d), 0.0001d); + Assert.AreEqual(true_cdf, U.CDF(0.9d), 0.0001d); + Assert.AreEqual(true_icdf, U.InverseCDF(true_cdf), 0.0001d); } /// @@ -105,11 +105,11 @@ public void Test_Uniform_R() public void Test_Construction() { var U = new Uniform(); - Assert.AreEqual(0,U.Min); + Assert.AreEqual(0, U.Min); Assert.AreEqual(1, U.Max); var U2 = new Uniform(2,10); - Assert.AreEqual(2, U2.Min); + Assert.AreEqual(2, U2.Min); Assert.AreEqual(10, U2.Max); } @@ -142,7 +142,7 @@ public void Test_InvalidParameters() public void Test_ParametersToString() { var U = new Uniform(); - Assert.AreEqual("Min",U.ParametersToString[0, 0]); + Assert.AreEqual("Min", U.ParametersToString[0, 0]); Assert.AreEqual("Max", U.ParametersToString[1, 0]); Assert.AreEqual("0", U.ParametersToString[0, 1]); Assert.AreEqual("1", U.ParametersToString[1, 1]); @@ -156,10 +156,10 @@ public void Test_Moments() { var dist = new Uniform(2, 10); var mom = dist.CentralMoments(1E-8); - Assert.AreEqual(mom[0], dist.Mean, 1E-2); - Assert.AreEqual(mom[1], dist.StandardDeviation, 1E-2); - Assert.AreEqual(mom[2], dist.Skewness, 1E-2); - Assert.AreEqual(mom[3], dist.Kurtosis, 1E-2); + Assert.AreEqual(dist.Mean, mom[0], 1E-2); + Assert.AreEqual(dist.StandardDeviation, mom[1], 1E-2); + Assert.AreEqual(dist.Skewness, mom[2], 1E-2); + Assert.AreEqual(dist.Kurtosis, mom[3], 1E-2); } /// @@ -195,7 +195,7 @@ public void Test_Median() public void Test_Mode() { var U = new Uniform(); - Assert.AreEqual(double.NaN,U.Mode); + Assert.AreEqual(double.NaN, U.Mode); var U2 = new Uniform(2, 10); Assert.AreEqual(double.NaN, U2.Mode); @@ -208,7 +208,7 @@ public void Test_Mode() public void Test_StandardDeviation() { var U = new Uniform(); - Assert.AreEqual(0.288675, U.StandardDeviation, 1e-05); + Assert.AreEqual(0.288675, U.StandardDeviation, 1e-05); var U2 = new Uniform(2, 10); Assert.AreEqual(2.3094, U2.StandardDeviation, 1e-04); @@ -234,7 +234,7 @@ public void Test_Skewness() public void Test_Kurtosis() { var U = new Uniform(); - Assert.AreEqual(9d / 5d,U.Kurtosis); + Assert.AreEqual(9d / 5d, U.Kurtosis); var U2 = new Uniform(2, 10); Assert.AreEqual(9d / 5d, U2.Kurtosis); @@ -274,9 +274,9 @@ public void Test_PDF() public void Test_CDF() { var U = new Uniform(); - Assert.AreEqual(0,U.CDF(0)); - Assert.AreEqual(1,U.CDF(1)); - Assert.AreEqual(0.5,U.CDF(0.5)); + Assert.AreEqual(0, U.CDF(0)); + Assert.AreEqual(1, U.CDF(1)); + Assert.AreEqual(0.5, U.CDF(0.5)); } /// @@ -286,9 +286,9 @@ public void Test_CDF() public void Test_InverseCDF() { var U = new Uniform(); - Assert.AreEqual(0,U.InverseCDF(0)); - Assert.AreEqual(1,U.InverseCDF(1)); - Assert.AreEqual(0.3,U.InverseCDF(0.3)); + Assert.AreEqual(0, U.InverseCDF(0)); + Assert.AreEqual(1, U.InverseCDF(1)); + Assert.AreEqual(0.3, U.InverseCDF(0.3)); } } } diff --git a/Test_Numerics/Distributions/Univariate/Test_UniformDiscrete.cs b/Test_Numerics/Distributions/Univariate/Test_UniformDiscrete.cs index 60264d36..70301e9e 100644 --- a/Test_Numerics/Distributions/Univariate/Test_UniformDiscrete.cs +++ b/Test_Numerics/Distributions/Univariate/Test_UniformDiscrete.cs @@ -64,7 +64,8 @@ public void Test_UniformDiscreteDist() { double true_mean = 4.0d; double true_median = 4.0d; - double true_stdDev = Math.Sqrt(1.3333333333333333d); + // scipy.stats.randint(2, 7).std() = 1.4142135624; N=5, sqrt((N^2-1)/12) + double true_stdDev = Math.Sqrt((5.0d * 5.0d - 1.0d) / 12.0d); int true_skew = 0; double true_kurt = 1.7d; double true_pdf = 0.2d; @@ -72,15 +73,15 @@ public void Test_UniformDiscreteDist() double true_icdf05 = 2.0d; double true_icdf95 = 6.0d; var U = new UniformDiscrete(2d, 6d); - Assert.AreEqual(U.Mean, true_mean, 0.0001d); - Assert.AreEqual(U.Median, true_median, 0.0001d); - Assert.AreEqual(U.StandardDeviation, true_stdDev, 0.0001d); - Assert.AreEqual(U.Skewness, true_skew, 0.0001d); - Assert.AreEqual(U.Kurtosis, true_kurt, 0.0001d); - Assert.AreEqual(U.PDF(4.0d), true_pdf, 0.0001d); - Assert.AreEqual(U.CDF(2.0d), true_cdf, 0.0001d); - Assert.AreEqual(U.InverseCDF(0.17d), true_icdf05, 0.0001d); - Assert.AreEqual(U.InverseCDF(0.87d), true_icdf95, 0.0001d); + Assert.AreEqual(true_mean, U.Mean, 0.0001d); + Assert.AreEqual(true_median, U.Median, 0.0001d); + Assert.AreEqual(true_stdDev, U.StandardDeviation, 0.0001d); + Assert.AreEqual(true_skew, U.Skewness, 0.0001d); + Assert.AreEqual(true_kurt, U.Kurtosis, 0.0001d); + Assert.AreEqual(true_pdf, U.PDF(4.0d), 0.0001d); + Assert.AreEqual(true_cdf, U.CDF(2.0d), 0.0001d); + Assert.AreEqual(true_icdf05, U.InverseCDF(0.17d), 0.0001d); + Assert.AreEqual(true_icdf95, U.InverseCDF(0.87d), 0.0001d); } /// @@ -90,8 +91,8 @@ public void Test_UniformDiscreteDist() public void Test_Construction() { var U = new UniformDiscrete(); - Assert.AreEqual(0,U.Min); - Assert.AreEqual(1,U.Max); + Assert.AreEqual(0, U.Min); + Assert.AreEqual(1, U.Max); var U2 = new UniformDiscrete(2, 10); Assert.AreEqual(2, U2.Min); @@ -127,7 +128,7 @@ public void Test_InvalidParameters() public void Test_ParametersToString() { var U = new UniformDiscrete(); - Assert.AreEqual("Min",U.ParametersToString[0, 0] ); + Assert.AreEqual("Min", U.ParametersToString[0, 0]); Assert.AreEqual("Max", U.ParametersToString[1, 0]); Assert.AreEqual("0", U.ParametersToString[0, 1]); Assert.AreEqual("1", U.ParametersToString[1, 1]); @@ -166,7 +167,7 @@ public void Test_Median() public void Test_Mode() { var U = new UniformDiscrete(); - Assert.AreEqual(double.NaN,U.Mode); + Assert.AreEqual(double.NaN, U.Mode); var U2 = new UniformDiscrete(2, 10); Assert.AreEqual(double.NaN, U2.Mode); @@ -178,11 +179,13 @@ public void Test_Mode() [TestMethod] public void Test_StandardDeviation() { + // scipy.stats.randint(0, 2).std() = 0.5; N=2, sqrt((4-1)/12) = 0.5 var U = new UniformDiscrete(); - Assert.AreEqual(0.288675, U.StandardDeviation, 1e-05); + Assert.AreEqual(0.5, U.StandardDeviation, 1e-05); + // scipy.stats.randint(2, 11).std() = 2.5819888975; N=9, sqrt((81-1)/12) var U2 = new UniformDiscrete(2, 10); - Assert.AreEqual(2.3094, U2.StandardDeviation, 1e-04); + Assert.AreEqual(2.5819888975, U2.StandardDeviation, 1e-04); } /// @@ -258,8 +261,36 @@ public void Test_InverseCDF() { var U = new UniformDiscrete(); Assert.AreEqual(0, U.InverseCDF(0)); - Assert.AreEqual(1,U.InverseCDF(1)); - Assert.AreEqual(0,U.InverseCDF(0.3)); + Assert.AreEqual(1, U.InverseCDF(1)); + Assert.AreEqual(0, U.InverseCDF(0.3)); + } + + /// + /// Verify discrete uniform standard deviation uses the correct formula: sqrt((N^2-1)/12). + /// Reference: scipy.stats.randint(low, high+1).std() + /// + [TestMethod()] + public void Test_StandardDeviation_Discrete() + { + // randint(0, 2).std() = 0.5000000000 {0,1} + var u1 = new UniformDiscrete(0, 1); + Assert.AreEqual(0.5, u1.StandardDeviation, 1E-6); + + // randint(0, 6).std() = 1.7078251277 {0,...,5} + var u2 = new UniformDiscrete(0, 5); + Assert.AreEqual(1.7078251277, u2.StandardDeviation, 1E-6); + + // randint(0, 11).std() = 3.1622776602 {0,...,10} + var u3 = new UniformDiscrete(0, 10); + Assert.AreEqual(3.1622776602, u3.StandardDeviation, 1E-6); + + // randint(1, 7).std() = 1.7078251277 {1,...,6} + var u4 = new UniformDiscrete(1, 6); + Assert.AreEqual(1.7078251277, u4.StandardDeviation, 1E-6); + + // randint(3, 8).std() = 1.4142135624 {3,...,7} + var u5 = new UniformDiscrete(3, 7); + Assert.AreEqual(1.4142135624, u5.StandardDeviation, 1E-6); } } diff --git a/Test_Numerics/Distributions/Univariate/Test_VonMises.cs b/Test_Numerics/Distributions/Univariate/Test_VonMises.cs new file mode 100644 index 00000000..cfe1b978 --- /dev/null +++ b/Test_Numerics/Distributions/Univariate/Test_VonMises.cs @@ -0,0 +1,313 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Distributions; +using Numerics.Mathematics.SpecialFunctions; + +namespace Distributions.Univariate +{ + /// + /// Unit tests for the Von Mises distribution. + /// + /// + /// + /// Authors: + /// + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + /// References: + /// + /// + /// Reference values computed using R package 'circular' (dvonmises, pvonmises) and scipy.stats.vonmises. + /// + /// + [TestClass] + public class Test_VonMises + { + + /// + /// Verifying default construction. + /// + [TestMethod] + public void Test_Construction() + { + var VM = new VonMises(); + Assert.AreEqual(0d, VM.Mu); + Assert.AreEqual(1d, VM.Kappa); + + var VM2 = new VonMises(1.0, 2.0); + Assert.AreEqual(1.0, VM2.Mu); + Assert.AreEqual(2.0, VM2.Kappa); + } + + /// + /// Testing distribution with bad parameters. + /// + [TestMethod] + public void Test_InvalidParameters() + { + var VM = new VonMises(double.NaN, double.NaN); + Assert.IsFalse(VM.ParametersValid); + + var VM2 = new VonMises(0, -1); + Assert.IsFalse(VM2.ParametersValid); + + var VM3 = new VonMises(5, 1); // mu > pi + Assert.IsFalse(VM3.ParametersValid); + } + + /// + /// Testing ParametersToString(). + /// + [TestMethod] + public void Test_ParametersToString() + { + var VM = new VonMises(); + Assert.AreEqual("Mean Direction (μ)", VM.ParametersToString[0, 0]); + Assert.AreEqual("Concentration (κ)", VM.ParametersToString[1, 0]); + } + + /// + /// Testing the range is [-π, π]. + /// + [TestMethod] + public void Test_MinMax() + { + var VM = new VonMises(); + Assert.AreEqual(-Math.PI, VM.Minimum); + Assert.AreEqual(Math.PI, VM.Maximum); + } + + /// + /// Testing mean, median, and mode. + /// + [TestMethod] + public void Test_Mean() + { + var VM = new VonMises(0.5, 2.0); + Assert.AreEqual(0.5, VM.Mean); + Assert.AreEqual(0.5, VM.Median); + Assert.AreEqual(0.5, VM.Mode); + } + + /// + /// Test the PDF using the analytical formula: f(x) = exp(κ cos(x - μ)) / (2π I₀(κ)). + /// + /// + /// Verified against scipy.stats.vonmises.pdf(x, kappa, loc=mu): + /// VM(0,1): PDF(0) = exp(1)/(2π·I₀(1)) ≈ 0.34171 + /// VM(0,1): PDF(π/2) = 1/(2π·I₀(1)) ≈ 0.12573 + /// VM(0,1): PDF(π) = exp(-1)/(2π·I₀(1)) ≈ 0.04626 + /// + [TestMethod] + public void Test_PDF() + { + // VM(mu=0, kappa=1): analytical values + var VM = new VonMises(0, 1); + double i0_1 = Bessel.I0(1); // ≈ 1.2660658 + double norm = 2d * Math.PI * i0_1; + + Assert.AreEqual(Math.Exp(1d) / norm, VM.PDF(0), 1e-6); // exp(cos(0)) / norm + Assert.AreEqual(1d / norm, VM.PDF(Math.PI / 2), 1e-6); // exp(cos(π/2)) / norm = exp(0) / norm + Assert.AreEqual(Math.Exp(-1d) / norm, VM.PDF(Math.PI), 1e-6); // exp(cos(π)) / norm + + // PDF should be symmetric about mu + Assert.AreEqual(VM.PDF(0.5), VM.PDF(-0.5), 1e-10); + + // VM(mu=0, kappa=5): more concentrated + var VM2 = new VonMises(0, 5); + double i0_5 = Bessel.I0(5); + Assert.AreEqual(Math.Exp(5d) / (2d * Math.PI * i0_5), VM2.PDF(0), 1e-6); + + // PDF outside [-π, π] should be 0 + Assert.AreEqual(0d, VM.PDF(-4.0)); + } + + /// + /// Test the CDF boundary conditions and symmetry. + /// + [TestMethod] + public void Test_CDF() + { + var VM = new VonMises(0, 1); + + // Boundary conditions + Assert.AreEqual(0d, VM.CDF(-Math.PI), 1e-6); + Assert.AreEqual(1d, VM.CDF(Math.PI), 1e-6); + + // CDF at mean direction should be 0.5 (symmetry) + Assert.AreEqual(0.5, VM.CDF(0), 1e-4); + + // CDF should be monotonically increasing + double prev = 0; + for (double x = -Math.PI; x <= Math.PI; x += 0.1) + { + double cdf = VM.CDF(x); + Assert.IsGreaterThanOrEqualTo(prev - 1e-10, cdf); + prev = cdf; + } + + // CDF at pi/2 should be > 0.5 (right of mean) + double cdfPiHalf = VM.CDF(Math.PI / 2); + Assert.IsGreaterThan(0.5, cdfPiHalf); + Assert.IsLessThan(1.0, cdfPiHalf); + } + + /// + /// Test InverseCDF is consistent with CDF. + /// + [TestMethod] + public void Test_InverseCDF() + { + var VM = new VonMises(0, 2); + Assert.AreEqual(-Math.PI, VM.InverseCDF(0)); + Assert.AreEqual(Math.PI, VM.InverseCDF(1)); + + // CDF-InverseCDF round-trip + double[] probs = { 0.1, 0.25, 0.5, 0.75, 0.9 }; + foreach (var p in probs) + { + double x = VM.InverseCDF(p); + double pBack = VM.CDF(x); + Assert.AreEqual(p, pBack, 1e-4); + } + } + + /// + /// Test the MLE estimation with known data. + /// + [TestMethod] + public void Test_MLE() + { + // Generate a sample from VM(mu=1.0, kappa=3.0) and verify MLE recovers parameters + var VM = new VonMises(1.0, 3.0); + var sample = VM.GenerateRandomValues(5000, seed: 12345); + + var fitted = new VonMises(); + fitted.Estimate(sample, ParameterEstimationMethod.MaximumLikelihood); + + Assert.AreEqual(1.0, fitted.Mu, 0.1); + Assert.AreEqual(3.0, fitted.Kappa, 0.3); + } + + /// + /// Test the circular variance. + /// + /// + /// Circular variance = 1 - I1(kappa)/I0(kappa). + /// For kappa=0: variance = 1 (uniform on circle). + /// For kappa→∞: variance → 0 (all mass at mu). + /// For kappa=1: A(1) = I1(1)/I0(1) ≈ 0.44606, so V ≈ 0.55394. + /// + [TestMethod] + public void Test_Variance() + { + // kappa = 0 → uniform on circle → variance = 1 + var VM0 = new VonMises(0, 0); + Assert.AreEqual(1.0, VM0.Variance, 1e-6); + + // kappa = 1 → variance ≈ 0.55394 + var VM1 = new VonMises(0, 1); + Assert.AreEqual(0.55394, VM1.Variance, 1e-3); + } + + /// + /// Test the Bessel function I0 against known values. + /// + [TestMethod] + public void Test_BesselI0() + { + // I0(0) = 1 + Assert.AreEqual(1.0, Bessel.I0(0), 1e-6); + // I0(1) ≈ 1.2660658 + Assert.AreEqual(1.2660658, Bessel.I0(1), 1e-5); + // I0(2) ≈ 2.2795853 + Assert.AreEqual(2.2795853, Bessel.I0(2), 1e-5); + // I0(5) ≈ 27.239872 + Assert.AreEqual(27.239872, Bessel.I0(5), 1e-3); + } + + /// + /// Test the Bessel function I1 against known values. + /// + [TestMethod] + public void Test_BesselI1() + { + // I1(0) = 0 + Assert.AreEqual(0.0, Bessel.I1(0), 1e-6); + // I1(1) ≈ 0.5651591 + Assert.AreEqual(0.5651591, Bessel.I1(1), 1e-5); + // I1(2) ≈ 1.5906369 + Assert.AreEqual(1.5906369, Bessel.I1(2), 1e-5); + } + + /// + /// Test random sampling produces values in [-π, π] and recovers mean direction. + /// + [TestMethod] + public void Test_RandomSampling() + { + var VM = new VonMises(0.5, 5.0); + var samples = VM.GenerateRandomValues(10000, seed: 42); + + // All values should be in [-π, π] + foreach (var s in samples) + { + Assert.IsGreaterThanOrEqualTo(-Math.PI, s); + Assert.IsLessThanOrEqualTo(Math.PI, s); + } + + // Mean direction should be close to mu + double sumSin = 0, sumCos = 0; + foreach (var s in samples) + { + sumSin += Math.Sin(s); + sumCos += Math.Cos(s); + } + double meanDir = Math.Atan2(sumSin, sumCos); + Assert.AreEqual(0.5, meanDir, 0.05); + } + + /// + /// Test bootstrap method. + /// + [TestMethod] + public void Test_Bootstrap() + { + var VM = new VonMises(0, 2); + var bootstrapped = VM.Bootstrap(ParameterEstimationMethod.MaximumLikelihood, 500, seed: 123); + Assert.IsTrue(bootstrapped.ParametersValid); + } + } +} diff --git a/Test_Numerics/Distributions/Univariate/Test_Weibull.cs b/Test_Numerics/Distributions/Univariate/Test_Weibull.cs index 79284819..497b92b5 100644 --- a/Test_Numerics/Distributions/Univariate/Test_Weibull.cs +++ b/Test_Numerics/Distributions/Univariate/Test_Weibull.cs @@ -71,8 +71,8 @@ public void Test_Weibull_MLE_Fit() double kappa = W.Kappa; double true_L = 9.589d; double true_k = 1.907d; - Assert.IsLessThan(0.01d,(lamda - true_L) / true_L); - Assert.IsLessThan(0.01d,(kappa - true_k) / true_k); + Assert.IsLessThan(0.01d, (lamda - true_L) / true_L); + Assert.IsLessThan(0.01d, (kappa - true_k) / true_k); } /// @@ -148,7 +148,7 @@ public void Test_Weibull_GOF() public void Test_Construction() { var W = new Weibull(); - Assert.AreEqual(10,W.Lambda); + Assert.AreEqual(10, W.Lambda); Assert.AreEqual(2, W.Kappa); var W2 = new Weibull(1, 1); @@ -182,7 +182,7 @@ public void Test_InvalidParameters() public void Test_ParametersToString() { var W = new Weibull(); - Assert.AreEqual("Scale (λ)",W.ParametersToString[0, 0] ); + Assert.AreEqual("Scale (λ)", W.ParametersToString[0, 0]); Assert.AreEqual("Shape (κ)", W.ParametersToString[1, 0]); Assert.AreEqual("10", W.ParametersToString[0, 1]); Assert.AreEqual("2", W.ParametersToString[1, 1]); @@ -222,10 +222,10 @@ public void Test_Mean() public void Test_Median() { var W = new Weibull(0.1, 1); - Assert.AreEqual(0.06931, W.Median, 1e-04); + Assert.AreEqual(0.06931, W.Median, 1e-04); var W2 = new Weibull(1, 1); - Assert.AreEqual(0.69314, W2.Median, 1e-04); + Assert.AreEqual(0.69314, W2.Median, 1e-04); } /// @@ -238,7 +238,7 @@ public void Test_Mode() Assert.AreEqual(0, W.Mode); var W2 = new Weibull(10, 10); - Assert.AreEqual(9.89519, W2.Mode, 1e-05); + Assert.AreEqual(9.89519, W2.Mode, 1e-05); } /// @@ -274,7 +274,7 @@ public void Test_Skewness() public void Test_Kurtosis() { var W = new Weibull(); - Assert.AreEqual(3.24508, W.Kurtosis,1e-04); + Assert.AreEqual(3.24508, W.Kurtosis, 1e-04); var W2 = new Weibull(1, 1); Assert.AreEqual(9, W2.Kurtosis); @@ -288,7 +288,7 @@ public void Test_MinMax() { var W = new Weibull(); Assert.AreEqual(0, W.Minimum); - Assert.AreEqual(double.PositiveInfinity,W.Maximum); + Assert.AreEqual(double.PositiveInfinity, W.Maximum); } /// @@ -299,7 +299,7 @@ public void Test_PDF() { var W = new Weibull(1, 1); Assert.AreEqual(1, W.PDF(0)); - Assert.AreEqual(0.36787, W.PDF(1), 1e-05); + Assert.AreEqual(0.36787, W.PDF(1), 1e-05); Assert.AreEqual(0.00004539, W.PDF(10), 1e-08); } @@ -322,9 +322,9 @@ public void Test_CDF() public void Test_InverseCDF() { var W = new Weibull(); - Assert.AreEqual(0,W.InverseCDF(0)); - Assert.AreEqual(double.PositiveInfinity,W.InverseCDF(1)); - Assert.AreEqual(7.1472, W.InverseCDF(0.4), 1e-04); + Assert.AreEqual(0, W.InverseCDF(0)); + Assert.AreEqual(double.PositiveInfinity, W.InverseCDF(1)); + Assert.AreEqual(7.1472, W.InverseCDF(0.4), 1e-04); } } } diff --git a/Test_Numerics/Functions/Test_Functions.cs b/Test_Numerics/Functions/Test_Functions.cs index 06cf0a27..b0f163df 100644 --- a/Test_Numerics/Functions/Test_Functions.cs +++ b/Test_Numerics/Functions/Test_Functions.cs @@ -232,7 +232,7 @@ public void Test_Tabular_Function() // Given X double X = 50.0; double Y = func.Function(X); - Assert.AreEqual(100.0,Y); + Assert.AreEqual(100.0, Y); // Given Y double Y2 = 100d; diff --git a/Test_Numerics/Functions/Test_LinkFunctions.cs b/Test_Numerics/Functions/Test_LinkFunctions.cs new file mode 100644 index 00000000..82ea0995 --- /dev/null +++ b/Test_Numerics/Functions/Test_LinkFunctions.cs @@ -0,0 +1,908 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using System.Xml.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Functions; + +namespace Functions +{ + /// + /// Unit tests for the link function classes: ILinkFunction implementations, + /// LinkController, LinkFunctionFactory, and LinkFunctionType. + /// + /// + /// Authors: + /// + /// + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + [TestClass] + public class Test_LinkFunctions + { + /// + /// Finite-difference step size for derivative verification. + /// + private const double DeltaH = 1e-7; + + /// + /// Tolerance for round-trip (Link then InverseLink) identity tests. + /// + private const double RoundTripTol = 1E-10; + + /// + /// Tolerance for derivative (finite-difference vs analytic) tests. + /// + private const double DerivativeTol = 1E-5; + + // ────────────────────────────────────────────── + // IdentityLink + // ────────────────────────────────────────────── + + /// + /// Test that IdentityLink.Link returns x unchanged. + /// + [TestMethod] + public void Test_IdentityLink_Link() + { + var link = new IdentityLink(); + double[] values = { -100.0, -1.5, 0.0, 1.5, 100.0, double.MaxValue, double.MinValue }; + foreach (double x in values) + { + Assert.AreEqual(x, link.Link(x), 0.0); + } + } + + /// + /// Test that IdentityLink.InverseLink returns eta unchanged. + /// + [TestMethod] + public void Test_IdentityLink_InverseLink() + { + var link = new IdentityLink(); + double[] values = { -100.0, -1.5, 0.0, 1.5, 100.0 }; + foreach (double eta in values) + { + Assert.AreEqual(eta, link.InverseLink(eta), 0.0); + } + } + + /// + /// Test that IdentityLink.DLink always returns 1. + /// + [TestMethod] + public void Test_IdentityLink_DLink() + { + var link = new IdentityLink(); + double[] values = { -100.0, 0.0, 100.0, 42.0 }; + foreach (double x in values) + { + Assert.AreEqual(1.0, link.DLink(x), 0.0); + } + } + + /// + /// Test round-trip: InverseLink(Link(x)) == x for IdentityLink. + /// + [TestMethod] + public void Test_IdentityLink_RoundTrip() + { + var link = new IdentityLink(); + double[] values = { -1000.0, -1.0, 0.0, 1.0, 1000.0 }; + foreach (double x in values) + { + double recovered = link.InverseLink(link.Link(x)); + Assert.AreEqual(x, recovered, RoundTripTol); + } + } + + // ────────────────────────────────────────────── + // LogLink + // ────────────────────────────────────────────── + + /// + /// Test LogLink.Link against known values. + /// + [TestMethod] + public void Test_LogLink_Link_KnownValues() + { + var link = new LogLink(); + Assert.AreEqual(0.0, link.Link(1.0), 1E-12); + Assert.AreEqual(Math.Log(2.0), link.Link(2.0), 1E-12); + Assert.AreEqual(Math.Log(10.0), link.Link(10.0), 1E-12); + Assert.AreEqual(Math.Log(0.5), link.Link(0.5), 1E-12); + } + + /// + /// Test LogLink.InverseLink against known values. + /// + [TestMethod] + public void Test_LogLink_InverseLink_KnownValues() + { + var link = new LogLink(); + Assert.AreEqual(1.0, link.InverseLink(0.0), 1E-12); + Assert.AreEqual(Math.E, link.InverseLink(1.0), 1E-12); + Assert.AreEqual(Math.Exp(2.0), link.InverseLink(2.0), 1E-12); + Assert.AreEqual(Math.Exp(-1.0), link.InverseLink(-1.0), 1E-12); + } + + /// + /// Test LogLink.DLink against known values: h'(x) = 1/x. + /// + [TestMethod] + public void Test_LogLink_DLink_KnownValues() + { + var link = new LogLink(); + Assert.AreEqual(1.0, link.DLink(1.0), 1E-12); + Assert.AreEqual(0.5, link.DLink(2.0), 1E-12); + Assert.AreEqual(0.1, link.DLink(10.0), 1E-12); + Assert.AreEqual(100.0, link.DLink(0.01), 1E-6); + } + + /// + /// Test round-trip: InverseLink(Link(x)) == x for LogLink over positive domain. + /// + [TestMethod] + public void Test_LogLink_RoundTrip() + { + var link = new LogLink(); + double[] values = { 0.001, 0.1, 1.0, 10.0, 100.0, 1e6 }; + foreach (double x in values) + { + double recovered = link.InverseLink(link.Link(x)); + Assert.AreEqual(x, recovered, x * 1E-10); + } + } + + /// + /// Test LogLink.Link throws for non-positive x. + /// + [TestMethod] + public void Test_LogLink_Link_ThrowsForZero() + { + Assert.Throws(() => new LogLink().Link(0.0)); + } + + /// + /// Test LogLink.Link throws for negative x. + /// + [TestMethod] + public void Test_LogLink_Link_ThrowsForNegative() + { + Assert.Throws(() => new LogLink().Link(-1.0)); + } + + /// + /// Test LogLink.DLink throws for non-positive x. + /// + [TestMethod] + public void Test_LogLink_DLink_ThrowsForZero() + { + Assert.Throws(() => new LogLink().DLink(0.0)); + } + + /// + /// Test LogLink derivative via finite difference. + /// + [TestMethod] + public void Test_LogLink_DerivativeConsistency() + { + var link = new LogLink(); + double[] testPoints = { 0.01, 0.5, 1.0, 5.0, 100.0 }; + foreach (double x in testPoints) + { + double finiteDiff = (link.Link(x + DeltaH) - link.Link(x - DeltaH)) / (2 * DeltaH); + Assert.AreEqual(finiteDiff, link.DLink(x), DerivativeTol); + } + } + + // ────────────────────────────────────────────── + // LogitLink + // ────────────────────────────────────────────── + + /// + /// Test LogitLink.Link against known values: logit(0.5) = 0. + /// + [TestMethod] + public void Test_LogitLink_Link_KnownValues() + { + var link = new LogitLink(); + Assert.AreEqual(0.0, link.Link(0.5), 1E-12); + // logit(0.731) = log(0.731/0.269) ≈ 0.99894 + Assert.AreEqual(Math.Log(0.731 / 0.269), link.Link(0.731), 1E-6); + // logit(0.1) = log(1/9) ≈ -2.19722 + Assert.AreEqual(Math.Log(1.0 / 9.0), link.Link(0.1), 1E-10); + } + + /// + /// Test LogitLink.InverseLink (sigmoid) against known values. + /// + [TestMethod] + public void Test_LogitLink_InverseLink_KnownValues() + { + var link = new LogitLink(); + Assert.AreEqual(0.5, link.InverseLink(0.0), 1E-12); + // sigmoid(large positive) ≈ 1 + Assert.AreEqual(1.0, link.InverseLink(100.0), 1E-10); + // sigmoid(large negative) ≈ 0 + Assert.AreEqual(0.0, link.InverseLink(-100.0), 1E-10); + // sigmoid(1) = 1/(1+e^{-1}) ≈ 0.7310586 + Assert.AreEqual(1.0 / (1.0 + Math.Exp(-1.0)), link.InverseLink(1.0), 1E-10); + } + + /// + /// Test LogitLink.DLink against known values: h'(x) = 1/(x(1-x)). + /// + [TestMethod] + public void Test_LogitLink_DLink_KnownValues() + { + var link = new LogitLink(); + // At x=0.5: h'(0.5) = 1/(0.5*0.5) = 4.0 + Assert.AreEqual(4.0, link.DLink(0.5), 1E-12); + // At x=0.1: h'(0.1) = 1/(0.1*0.9) = 100/9 ≈ 11.1111 + Assert.AreEqual(1.0 / (0.1 * 0.9), link.DLink(0.1), 1E-10); + } + + /// + /// Test round-trip: InverseLink(Link(x)) == x for LogitLink over (0,1). + /// + [TestMethod] + public void Test_LogitLink_RoundTrip() + { + var link = new LogitLink(); + double[] values = { 0.001, 0.1, 0.25, 0.5, 0.75, 0.9, 0.999 }; + foreach (double x in values) + { + double recovered = link.InverseLink(link.Link(x)); + Assert.AreEqual(x, recovered, RoundTripTol); + } + } + + /// + /// Test LogitLink.Link throws for x outside (0,1). + /// + [TestMethod] + public void Test_LogitLink_Link_ThrowsForZero() + { + Assert.Throws(() => new LogitLink().Link(0.0)); + } + + /// + /// Test LogitLink.Link throws for x >= 1. + /// + [TestMethod] + public void Test_LogitLink_Link_ThrowsForOne() + { + Assert.Throws(() => new LogitLink().Link(1.0)); + } + + /// + /// Test LogitLink.Link throws for negative x. + /// + [TestMethod] + public void Test_LogitLink_Link_ThrowsForNegative() + { + Assert.Throws(() => new LogitLink().Link(-0.5)); + } + + /// + /// Test LogitLink sigmoid numerical stability for extreme eta values. + /// + [TestMethod] + public void Test_LogitLink_InverseLink_ExtremeValues() + { + var link = new LogitLink(); + // Very large positive eta: sigmoid should be very close to 1 without overflow + double high = link.InverseLink(710.0); + Assert.IsTrue(high > 0.999 && high <= 1.0); + // Very large negative eta: sigmoid should be very close to 0 without underflow + double low = link.InverseLink(-710.0); + Assert.IsTrue(low >= 0.0 && low < 0.001); + } + + /// + /// Test LogitLink derivative via finite difference. + /// + [TestMethod] + public void Test_LogitLink_DerivativeConsistency() + { + var link = new LogitLink(); + double[] testPoints = { 0.01, 0.1, 0.3, 0.5, 0.7, 0.9, 0.99 }; + foreach (double x in testPoints) + { + double finiteDiff = (link.Link(x + DeltaH) - link.Link(x - DeltaH)) / (2 * DeltaH); + Assert.AreEqual(finiteDiff, link.DLink(x), DerivativeTol); + } + } + + // ────────────────────────────────────────────── + // ProbitLink + // ────────────────────────────────────────────── + + /// + /// Test ProbitLink.Link against known values: Phi^{-1}(0.5) = 0. + /// + [TestMethod] + public void Test_ProbitLink_Link_KnownValues() + { + var link = new ProbitLink(); + Assert.AreEqual(0.0, link.Link(0.5), 1E-10); + // Phi^{-1}(0.975) ≈ 1.95996 + Assert.AreEqual(1.95996, link.Link(0.975), 1E-4); + // Phi^{-1}(0.025) ≈ -1.95996 + Assert.AreEqual(-1.95996, link.Link(0.025), 1E-4); + // Phi^{-1}(0.8413) ≈ 1.0 (CDF at z=1) + Assert.AreEqual(1.0, link.Link(0.8413), 1E-3); + } + + /// + /// Test ProbitLink.InverseLink against known values: Phi(0) = 0.5. + /// + [TestMethod] + public void Test_ProbitLink_InverseLink_KnownValues() + { + var link = new ProbitLink(); + Assert.AreEqual(0.5, link.InverseLink(0.0), 1E-10); + // Phi(1.96) ≈ 0.975 + Assert.AreEqual(0.975, link.InverseLink(1.96), 1E-3); + // Phi(-1.96) ≈ 0.025 + Assert.AreEqual(0.025, link.InverseLink(-1.96), 1E-3); + } + + /// + /// Test round-trip: InverseLink(Link(x)) == x for ProbitLink over (0,1). + /// + [TestMethod] + public void Test_ProbitLink_RoundTrip() + { + var link = new ProbitLink(); + double[] values = { 0.001, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.999 }; + foreach (double x in values) + { + double recovered = link.InverseLink(link.Link(x)); + Assert.AreEqual(x, recovered, RoundTripTol); + } + } + + /// + /// Test ProbitLink.Link throws for x outside (0,1). + /// + [TestMethod] + public void Test_ProbitLink_Link_ThrowsForZero() + { + Assert.Throws(() => new ProbitLink().Link(0.0)); + } + + /// + /// Test ProbitLink.Link throws for x >= 1. + /// + [TestMethod] + public void Test_ProbitLink_Link_ThrowsForOne() + { + Assert.Throws(() => new ProbitLink().Link(1.0)); + } + + /// + /// Test ProbitLink derivative via finite difference. + /// + [TestMethod] + public void Test_ProbitLink_DerivativeConsistency() + { + var link = new ProbitLink(); + double[] testPoints = { 0.05, 0.1, 0.3, 0.5, 0.7, 0.9, 0.95 }; + foreach (double x in testPoints) + { + double finiteDiff = (link.Link(x + DeltaH) - link.Link(x - DeltaH)) / (2 * DeltaH); + Assert.AreEqual(finiteDiff, link.DLink(x), DerivativeTol); + } + } + + // ────────────────────────────────────────────── + // ComplementaryLogLogLink + // ────────────────────────────────────────────── + + /// + /// Test CLogLog.Link against known values: h(x) = log(-log(1-x)). + /// + [TestMethod] + public void Test_CLogLog_Link_KnownValues() + { + var link = new ComplementaryLogLogLink(); + // h(1-1/e) = h(0.63212...) = log(-log(1/e)) = log(1) = 0 + double x0 = 1.0 - 1.0 / Math.E; + Assert.AreEqual(0.0, link.Link(x0), 1E-10); + // h(0.5) = log(-log(0.5)) = log(log(2)) ≈ -0.36651 + Assert.AreEqual(Math.Log(Math.Log(2.0)), link.Link(0.5), 1E-10); + } + + /// + /// Test CLogLog.InverseLink against known values: h^{-1}(eta) = 1-exp(-exp(eta)). + /// + [TestMethod] + public void Test_CLogLog_InverseLink_KnownValues() + { + var link = new ComplementaryLogLogLink(); + // h^{-1}(0) = 1 - exp(-1) = 1 - 1/e ≈ 0.63212 + Assert.AreEqual(1.0 - 1.0 / Math.E, link.InverseLink(0.0), 1E-10); + // For large positive eta: result approaches 1 + Assert.AreEqual(1.0, link.InverseLink(10.0), 1E-4); + // For large negative eta: result approaches 0 + Assert.AreEqual(0.0, link.InverseLink(-10.0), 1E-4); + } + + /// + /// Test round-trip: InverseLink(Link(x)) == x for CLogLog over (0,1). + /// + [TestMethod] + public void Test_CLogLog_RoundTrip() + { + var link = new ComplementaryLogLogLink(); + double[] values = { 0.001, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.999 }; + foreach (double x in values) + { + double recovered = link.InverseLink(link.Link(x)); + Assert.AreEqual(x, recovered, RoundTripTol); + } + } + + /// + /// Test CLogLog.Link throws for x outside (0,1). + /// + [TestMethod] + public void Test_CLogLog_Link_ThrowsForZero() + { + Assert.Throws(() => new ComplementaryLogLogLink().Link(0.0)); + } + + /// + /// Test CLogLog.Link throws for x >= 1. + /// + [TestMethod] + public void Test_CLogLog_Link_ThrowsForOne() + { + Assert.Throws(() => new ComplementaryLogLogLink().Link(1.0)); + } + + /// + /// Test CLogLog derivative via finite difference. + /// + [TestMethod] + public void Test_CLogLog_DerivativeConsistency() + { + var link = new ComplementaryLogLogLink(); + double[] testPoints = { 0.05, 0.1, 0.3, 0.5, 0.7, 0.9, 0.95 }; + foreach (double x in testPoints) + { + double finiteDiff = (link.Link(x + DeltaH) - link.Link(x - DeltaH)) / (2 * DeltaH); + Assert.AreEqual(finiteDiff, link.DLink(x), DerivativeTol); + } + } + + // ────────────────────────────────────────────── + // LinkFunctionFactory + // ────────────────────────────────────────────── + + /// + /// Test that factory creates the correct type for each enum value. + /// + [TestMethod] + public void Test_Factory_CreatesCorrectTypes() + { + Assert.IsInstanceOfType(LinkFunctionFactory.Create(LinkFunctionType.Identity), typeof(IdentityLink)); + Assert.IsInstanceOfType(LinkFunctionFactory.Create(LinkFunctionType.Log), typeof(LogLink)); + Assert.IsInstanceOfType(LinkFunctionFactory.Create(LinkFunctionType.Logit), typeof(LogitLink)); + Assert.IsInstanceOfType(LinkFunctionFactory.Create(LinkFunctionType.Probit), typeof(ProbitLink)); + Assert.IsInstanceOfType(LinkFunctionFactory.Create(LinkFunctionType.ComplementaryLogLog), typeof(ComplementaryLogLogLink)); + } + + /// + /// Test that factory throws for invalid enum value. + /// + [TestMethod] + public void Test_Factory_ThrowsForInvalidType() + { + Assert.Throws(() => LinkFunctionFactory.Create((LinkFunctionType)999)); + } + + /// + /// Test that factory-created links produce correct round-trip results. + /// + [TestMethod] + public void Test_Factory_RoundTripViaFactory() + { + // Identity: domain = all reals + var identity = LinkFunctionFactory.Create(LinkFunctionType.Identity); + Assert.AreEqual(5.0, identity.InverseLink(identity.Link(5.0)), RoundTripTol); + + // Log: domain = (0, inf) + var log = LinkFunctionFactory.Create(LinkFunctionType.Log); + Assert.AreEqual(2.5, log.InverseLink(log.Link(2.5)), RoundTripTol); + + // Logit: domain = (0, 1) + var logit = LinkFunctionFactory.Create(LinkFunctionType.Logit); + Assert.AreEqual(0.3, logit.InverseLink(logit.Link(0.3)), RoundTripTol); + + // Probit: domain = (0, 1) + var probit = LinkFunctionFactory.Create(LinkFunctionType.Probit); + Assert.AreEqual(0.7, probit.InverseLink(probit.Link(0.7)), RoundTripTol); + + // CLogLog: domain = (0, 1) + var cloglog = LinkFunctionFactory.Create(LinkFunctionType.ComplementaryLogLog); + Assert.AreEqual(0.4, cloglog.InverseLink(cloglog.Link(0.4)), RoundTripTol); + } + + // ────────────────────────────────────────────── + // LinkController + // ────────────────────────────────────────────── + + /// + /// Test that empty LinkController acts as identity for all parameters. + /// + [TestMethod] + public void Test_LinkController_Empty_ActsAsIdentity() + { + var ctrl = new LinkController(); + Assert.AreEqual(0, ctrl.Count); + + double[] x = { 100.0, 0.5, -0.1 }; + double[] eta = ctrl.Link(x); + Assert.AreEqual(100.0, eta[0], 1E-12); + Assert.AreEqual(0.5, eta[1], 1E-12); + Assert.AreEqual(-0.1, eta[2], 1E-12); + + double[] xBack = ctrl.InverseLink(eta); + Assert.AreEqual(100.0, xBack[0], 1E-12); + Assert.AreEqual(0.5, xBack[1], 1E-12); + Assert.AreEqual(-0.1, xBack[2], 1E-12); + } + + /// + /// Test LinkController with single LogLink on first parameter. + /// + [TestMethod] + public void Test_LinkController_SingleLink() + { + var ctrl = new LinkController(new LogLink()); + Assert.AreEqual(1, ctrl.Count); + + double[] x = { 2.0, 5.0, -3.0 }; + double[] eta = ctrl.Link(x); + // First element transformed by LogLink + Assert.AreEqual(Math.Log(2.0), eta[0], 1E-12); + // Remaining elements pass through (no link registered) + Assert.AreEqual(5.0, eta[1], 1E-12); + Assert.AreEqual(-3.0, eta[2], 1E-12); + } + + /// + /// Test LinkController with mixed links (null = identity). + /// + [TestMethod] + public void Test_LinkController_MixedLinks() + { + var ctrl = new LinkController(null, new LogLink(), null); + Assert.AreEqual(3, ctrl.Count); + + double[] x = { 42.0, 10.0, -7.0 }; + double[] eta = ctrl.Link(x); + Assert.AreEqual(42.0, eta[0], 1E-12); // null = identity + Assert.AreEqual(Math.Log(10.0), eta[1], 1E-12); // LogLink + Assert.AreEqual(-7.0, eta[2], 1E-12); // null = identity + + double[] xBack = ctrl.InverseLink(eta); + Assert.AreEqual(42.0, xBack[0], 1E-12); + Assert.AreEqual(10.0, xBack[1], 1E-10); + Assert.AreEqual(-7.0, xBack[2], 1E-12); + } + + /// + /// Test LinkController round-trip with location=null, scale=Log, shape=Logit. + /// + [TestMethod] + public void Test_LinkController_RoundTrip() + { + var ctrl = new LinkController(null, new LogLink(), new LogitLink()); + double[] x = { 100.0, 5.0, 0.3 }; + double[] eta = ctrl.Link(x); + double[] xBack = ctrl.InverseLink(eta); + Assert.AreEqual(x[0], xBack[0], RoundTripTol); + Assert.AreEqual(x[1], xBack[1], RoundTripTol); + Assert.AreEqual(x[2], xBack[2], RoundTripTol); + } + + /// + /// Test LinkController handles array longer than link count (extra elements pass through). + /// + [TestMethod] + public void Test_LinkController_ArrayLongerThanLinkCount() + { + var ctrl = new LinkController(new LogLink()); + double[] x = { 3.0, 7.0, 11.0, 13.0 }; + double[] eta = ctrl.Link(x); + Assert.AreEqual(Math.Log(3.0), eta[0], 1E-12); + Assert.AreEqual(7.0, eta[1], 1E-12); + Assert.AreEqual(11.0, eta[2], 1E-12); + Assert.AreEqual(13.0, eta[3], 1E-12); + } + + /// + /// Test LinkController indexer returns correct links and null for out-of-range. + /// + [TestMethod] + public void Test_LinkController_Indexer() + { + var logLink = new LogLink(); + var ctrl = new LinkController(null, logLink); + Assert.IsNull(ctrl[0]); + Assert.AreSame(logLink, ctrl[1]); + Assert.IsNull(ctrl[2]); // out of range + Assert.IsNull(ctrl[-1]); // negative index + } + + /// + /// Test ForLocationScaleShape factory method. + /// + [TestMethod] + public void Test_LinkController_ForLocationScaleShape() + { + var logLink = new LogLink(); + var ctrl = LinkController.ForLocationScaleShape(scaleLink: logLink); + Assert.AreEqual(3, ctrl.Count); + Assert.IsNull(ctrl[0]); + Assert.AreSame(logLink, ctrl[1]); + Assert.IsNull(ctrl[2]); + } + + /// + /// Test LinkJacobian returns correct diagonal matrix. + /// + [TestMethod] + public void Test_LinkController_LinkJacobian() + { + var ctrl = new LinkController(null, new LogLink()); + double[] x = { 42.0, 5.0 }; + var J = ctrl.LinkJacobian(x); + + // First element: identity => diagonal = 1.0 + Assert.AreEqual(1.0, J[0, 0], 1E-12); + // Second element: LogLink => h'(5) = 1/5 = 0.2 + Assert.AreEqual(0.2, J[1, 1], 1E-12); + // Off-diagonal = 0 + Assert.AreEqual(0.0, J[0, 1], 1E-12); + Assert.AreEqual(0.0, J[1, 0], 1E-12); + } + + /// + /// Test LogDetJacobian for a single LogLink. + /// For LogLink: deta/dtheta = 1/theta, so log|det J^{-1}| = -sum(-log|1/theta|) = log(theta). + /// + [TestMethod] + public void Test_LinkController_LogDetJacobian() + { + var ctrl = new LinkController(new LogLink()); + // phi = [log(5)] in link-space + double[] phi = { Math.Log(5.0) }; + double logDetJ = ctrl.LogDetJacobian(phi); + // InverseLink(log5) = 5; DLink(5) = 1/5 = 0.2 + // log|det J^{-1}| = -log|0.2| = log(5) ≈ 1.60944 + Assert.AreEqual(Math.Log(5.0), logDetJ, 1E-10); + } + + /// + /// Test LogDetJacobian with identity (empty controller) returns 0. + /// + [TestMethod] + public void Test_LinkController_LogDetJacobian_Identity() + { + var ctrl = new LinkController(); + double[] phi = { 1.0, 2.0, 3.0 }; + double logDetJ = ctrl.LogDetJacobian(phi); + Assert.AreEqual(0.0, logDetJ, 1E-12); + } + + /// + /// Test LogDetJacobian with multiple links. + /// + [TestMethod] + public void Test_LinkController_LogDetJacobian_MultipleLinks() + { + var ctrl = new LinkController(null, new LogLink(), new LogLink()); + // phi = [100, log(3), log(7)] + double[] phi = { 100.0, Math.Log(3.0), Math.Log(7.0) }; + double logDetJ = ctrl.LogDetJacobian(phi); + // index 0: null => no contribution (0) + // index 1: theta=3, deta/dtheta=1/3 => contribution = -log(1/3) = log(3) + // index 2: theta=7, deta/dtheta=1/7 => contribution = -log(1/7) = log(7) + double expected = Math.Log(3.0) + Math.Log(7.0); + Assert.AreEqual(expected, logDetJ, 1E-10); + } + + #region XElement Serialization + + /// + /// Test IdentityLink round-trip through ToXElement and CreateFromXElement. + /// + [TestMethod] + public void Test_IdentityLink_RoundTrip_ToXElement() + { + var original = new IdentityLink(); + var xml = original.ToXElement(); + Assert.AreEqual("IdentityLink", xml.Name.LocalName); + + var restored = LinkFunctionFactory.CreateFromXElement(xml); + Assert.IsInstanceOfType(restored, typeof(IdentityLink)); + Assert.AreEqual(original.Link(5.0), restored.Link(5.0), 1E-15); + Assert.AreEqual(original.InverseLink(-3.0), restored.InverseLink(-3.0), 1E-15); + } + + /// + /// Test LogLink round-trip through ToXElement and CreateFromXElement. + /// + [TestMethod] + public void Test_LogLink_RoundTrip_ToXElement() + { + var original = new LogLink(); + var xml = original.ToXElement(); + Assert.AreEqual("LogLink", xml.Name.LocalName); + + var restored = LinkFunctionFactory.CreateFromXElement(xml); + Assert.IsInstanceOfType(restored, typeof(LogLink)); + Assert.AreEqual(original.Link(2.5), restored.Link(2.5), 1E-15); + Assert.AreEqual(original.InverseLink(1.0), restored.InverseLink(1.0), 1E-15); + } + + /// + /// Test LogitLink round-trip through ToXElement and CreateFromXElement. + /// + [TestMethod] + public void Test_LogitLink_RoundTrip_ToXElement() + { + var original = new LogitLink(); + var xml = original.ToXElement(); + Assert.AreEqual("LogitLink", xml.Name.LocalName); + + var restored = LinkFunctionFactory.CreateFromXElement(xml); + Assert.IsInstanceOfType(restored, typeof(LogitLink)); + Assert.AreEqual(original.Link(0.7), restored.Link(0.7), 1E-15); + Assert.AreEqual(original.InverseLink(0.5), restored.InverseLink(0.5), 1E-15); + } + + /// + /// Test ProbitLink round-trip through ToXElement and CreateFromXElement. + /// + [TestMethod] + public void Test_ProbitLink_RoundTrip_ToXElement() + { + var original = new ProbitLink(); + var xml = original.ToXElement(); + Assert.AreEqual("ProbitLink", xml.Name.LocalName); + + var restored = LinkFunctionFactory.CreateFromXElement(xml); + Assert.IsInstanceOfType(restored, typeof(ProbitLink)); + Assert.AreEqual(original.Link(0.3), restored.Link(0.3), 1E-15); + Assert.AreEqual(original.InverseLink(-1.0), restored.InverseLink(-1.0), 1E-15); + } + + /// + /// Test ComplementaryLogLogLink round-trip through ToXElement and CreateFromXElement. + /// + [TestMethod] + public void Test_ComplementaryLogLogLink_RoundTrip_ToXElement() + { + var original = new ComplementaryLogLogLink(); + var xml = original.ToXElement(); + Assert.AreEqual("ComplementaryLogLogLink", xml.Name.LocalName); + + var restored = LinkFunctionFactory.CreateFromXElement(xml); + Assert.IsInstanceOfType(restored, typeof(ComplementaryLogLogLink)); + Assert.AreEqual(original.Link(0.5), restored.Link(0.5), 1E-15); + Assert.AreEqual(original.InverseLink(0.0), restored.InverseLink(0.0), 1E-15); + } + + /// + /// Test that CreateFromXElement throws NotSupportedException for unknown element names. + /// + [TestMethod] + public void Test_LinkFunctionFactory_CreateFromXElement_UnknownType_Throws() + { + var xml = new XElement("UnknownLink"); + Assert.Throws(() => LinkFunctionFactory.CreateFromXElement(xml)); + } + + /// + /// Test LinkController round-trip serialization with an empty controller. + /// + [TestMethod] + public void Test_LinkController_RoundTrip_Empty() + { + var original = new LinkController(); + var xml = original.ToXElement(); + Assert.AreEqual("LinkController", xml.Name.LocalName); + + var restored = new LinkController(xml); + Assert.AreEqual(0, restored.Count); + } + + /// + /// Test LinkController round-trip serialization with all three link slots populated. + /// + [TestMethod] + public void Test_LinkController_RoundTrip_AllLinks() + { + var original = new LinkController(new IdentityLink(), new LogLink(), new LogitLink()); + Assert.AreEqual(3, original.Count); + + var xml = original.ToXElement(); + var restored = new LinkController(xml); + + Assert.AreEqual(3, restored.Count); + Assert.IsInstanceOfType(restored[0], typeof(IdentityLink)); + Assert.IsInstanceOfType(restored[1], typeof(LogLink)); + Assert.IsInstanceOfType(restored[2], typeof(LogitLink)); + + // Verify functional equivalence + double[] x = { 5.0, 2.5, 0.7 }; + double[] originalLinked = original.Link(x); + double[] restoredLinked = restored.Link(x); + for (int i = 0; i < x.Length; i++) + Assert.AreEqual(originalLinked[i], restoredLinked[i], 1E-15); + } + + /// + /// Test LinkController round-trip with null slots preserved. + /// + [TestMethod] + public void Test_LinkController_RoundTrip_NullSlots() + { + var original = new LinkController(null, new LogLink(), null); + Assert.AreEqual(3, original.Count); + + var xml = original.ToXElement(); + var restored = new LinkController(xml); + + Assert.AreEqual(3, restored.Count); + Assert.IsNull(restored[0]); + Assert.IsInstanceOfType(restored[1], typeof(LogLink)); + Assert.IsNull(restored[2]); + + // Verify functional equivalence: index 0 and 2 pass through, index 1 uses log + double[] x = { 5.0, 2.5, 0.7 }; + double[] originalLinked = original.Link(x); + double[] restoredLinked = restored.Link(x); + for (int i = 0; i < x.Length; i++) + Assert.AreEqual(originalLinked[i], restoredLinked[i], 1E-15); + } + + #endregion + } +} diff --git a/Test_Numerics/Machine Learning/Supervised/Test_GeneralizedLinearModel.cs b/Test_Numerics/Machine Learning/Supervised/Test_GeneralizedLinearModel.cs index 5640948f..2a12aa42 100644 --- a/Test_Numerics/Machine Learning/Supervised/Test_GeneralizedLinearModel.cs +++ b/Test_Numerics/Machine Learning/Supervised/Test_GeneralizedLinearModel.cs @@ -30,6 +30,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Numerics.Data; +using Numerics.Functions; using Numerics.MachineLearning; using Numerics.Mathematics.LinearAlgebra; using Numerics; @@ -200,7 +201,7 @@ public void Test_Log() var y = new Vector(deaths) { Header = "admits" }; var x = new Matrix(list) { Header = new string[] { "drivers", "popden" } }; - var GLM = new GeneralizedLinearModel(x, y, true, GeneralizedLinearModel.LinkFunctionType.Log); + var GLM = new GeneralizedLinearModel(x, y, true, LinkFunctionType.Log); GLM.Train(); var par = GLM.Parameters; @@ -264,7 +265,7 @@ public void Test_Logistic() var x = new Matrix(list) { Header = new string[] { "gre", "gpa", "rank" } }; - var GLM = new GeneralizedLinearModel(x, y, true, GeneralizedLinearModel.LinkFunctionType.Logit); + var GLM = new GeneralizedLinearModel(x, y, true, LinkFunctionType.Logit); GLM.Train(); var par = GLM.Parameters; @@ -329,7 +330,7 @@ public void Test_Probit() var x = new Matrix(list) { Header = new string[] { "gre", "gpa", "rank" } }; - var GLM = new GeneralizedLinearModel(x, y, true, GeneralizedLinearModel.LinkFunctionType.Probit); + var GLM = new GeneralizedLinearModel(x, y, true, LinkFunctionType.Probit); GLM.Train(); var par = GLM.Parameters; @@ -396,7 +397,7 @@ public void Test_LogLog() var x = new Matrix(list) { Header = new string[] { "gre", "gpa", "rank" } }; - var GLM = new GeneralizedLinearModel(x, y, true, GeneralizedLinearModel.LinkFunctionType.ComplementaryLogLog); + var GLM = new GeneralizedLinearModel(x, y, true, LinkFunctionType.ComplementaryLogLog); GLM.Train(); var par = GLM.Parameters; diff --git a/Test_Numerics/Machine Learning/Supervised/Test_kNN.cs b/Test_Numerics/Machine Learning/Supervised/Test_kNN.cs index f62ed345..df011b72 100644 --- a/Test_Numerics/Machine Learning/Supervised/Test_kNN.cs +++ b/Test_Numerics/Machine Learning/Supervised/Test_kNN.cs @@ -166,5 +166,37 @@ public void Test_kNN_Regression() } + /// + /// Verify kNN GetNeighbors returns correct indices for multi-row test input. + /// After fix, each row's neighbors are preserved (not just the last row's). + /// + [TestMethod] + public void Test_GetNeighbors_MultiRow() + { + // 2D dataset: 12 points in two well-separated clusters + // Cluster A near (0,0): indices 0-5 + // Cluster B near (100,100): indices 6-11 + var x1 = new double[] { 0, 1, 0, 1, 2, 0, 100, 101, 100, 101, 102, 100 }; + var x2 = new double[] { 0, 0, 1, 1, 0, 2, 100, 100, 101, 101, 100, 102 }; + var y = new double[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1 }; + var X_train = new Matrix(new List { x1, x2 }); + var Y_train = new Vector(y); + + var knn = new KNearestNeighbors(X_train, Y_train, 2); + + // Multi-row query: two points near opposite clusters + var query = new double[,] { { 0.5, 0.5 }, { 100.5, 100.5 } }; + var neighbors = knn.GetNeighbors(query); + Assert.IsNotNull(neighbors); + + // First query (0.5, 0.5): nearest neighbors should be from cluster A (indices 0-5) + Assert.IsLessThan(6, neighbors[0], $"First query's nearest neighbor should be in cluster A, got index {neighbors[0]}"); + Assert.IsLessThan(6, neighbors[1], $"First query's 2nd nearest should be in cluster A, got index {neighbors[1]}"); + + // Second query (100.5, 100.5): nearest neighbors should be from cluster B (indices 6-11) + Assert.IsGreaterThanOrEqualTo(6, neighbors[2], $"Second query's nearest neighbor should be in cluster B, got index {neighbors[2]}"); + Assert.IsGreaterThanOrEqualTo(6, neighbors[3], $"Second query's 2nd nearest should be in cluster B, got index {neighbors[3]}"); + } + } } diff --git a/Test_Numerics/Mathematics/Integration/Test_Integration.cs b/Test_Numerics/Mathematics/Integration/Test_Integration.cs index 343ed5c9..8d77bca5 100644 --- a/Test_Numerics/Mathematics/Integration/Test_Integration.cs +++ b/Test_Numerics/Mathematics/Integration/Test_Integration.cs @@ -54,7 +54,26 @@ public void Test_GaussLegendre() } /// - /// Test Trapezoidal Rule Method. + /// Test 20-point Gauss-Legendre with polynomial, trigonometric, and logarithmic integrands. + /// + [TestMethod()] + public void Test_GaussLegendre20() + { + // x^3 over [0,1] = 0.25 (exact for degree <= 39) + double e1 = Numerics.Mathematics.Integration.Integration.GaussLegendre20(Integrands.FX3, 0d, 1d); + Assert.AreEqual(0.25d, e1, 1E-14); + + // Cosine over [0,1] = sin(1) + double e2 = Numerics.Mathematics.Integration.Integration.GaussLegendre20(Math.Cos, 0d, 1d); + Assert.AreEqual(Math.Sin(1d), e2, 1E-14); + + // log(x) over [1,2] = 2*ln(2) - 1 + double e3 = Numerics.Mathematics.Integration.Integration.GaussLegendre20(Math.Log, 1d, 2d); + Assert.AreEqual(2d * Math.Log(2d) - 1d, e3, 1E-14); + } + + /// + /// Test Trapezoidal Rule Method. /// [TestMethod()] public void Test_TrapezoidalRule() diff --git a/Test_Numerics/Mathematics/Linear Algebra/Test_GaussJordanElimination.cs b/Test_Numerics/Mathematics/Linear Algebra/Test_GaussJordanElimination.cs index 1de9a5f5..6736146d 100644 --- a/Test_Numerics/Mathematics/Linear Algebra/Test_GaussJordanElimination.cs +++ b/Test_Numerics/Mathematics/Linear Algebra/Test_GaussJordanElimination.cs @@ -59,7 +59,7 @@ public void Test_GaussJordanElim() for (int i = 0; i < A.NumberOfRows; i++) { for (int j = 0; j < A.NumberOfColumns - 1; j++) - Assert.AreEqual(A[i, j],true_IA[i, j]); + Assert.AreEqual(true_IA[i, j], A[i, j]); } /// Recreated Gauss Jordan test in R to compare the inverted A matrices. diff --git a/Test_Numerics/Mathematics/Optimization/Constrained/Test_AugmentedLagrange.cs b/Test_Numerics/Mathematics/Optimization/Constrained/Test_AugmentedLagrange.cs index 1644ce04..862ba7ab 100644 --- a/Test_Numerics/Mathematics/Optimization/Constrained/Test_AugmentedLagrange.cs +++ b/Test_Numerics/Mathematics/Optimization/Constrained/Test_AugmentedLagrange.cs @@ -183,5 +183,97 @@ public void Test_RosenbrockDisk() // Multiplier Assert.AreEqual(0d, solver.Mu[0]); } + /// + /// Tests AugmentedLagrange with mixed constraint types (equality + lesser-than + greater-than). + /// This previously caused IndexOutOfRangeException due to incorrect multiplier array indexing. + /// + /// + /// Minimize x² + y² subject to: + /// x + y = 4 (equality) + /// x ≤ 3 (lesser-than-or-equal) + /// y ≥ 0.5 (greater-than-or-equal) + /// + /// Analytical solution: x = 2, y = 2 (unconstrained on equality). + /// But with x ≤ 3 and y ≥ 0.5, the equality x+y=4 with min x²+y² gives x=2, y=2. + /// All constraints are satisfied at (2,2). + /// + [TestMethod] + public void Test_MixedConstraints() + { + // Objective: minimize x² + y² + Func func = (double[] x) => + { + return x[0] * x[0] + x[1] * x[1]; + }; + + // Constraints + var equalityConstraint = new Constraint( + (x) => x[0] + x[1], 2, 4.0, ConstraintType.EqualTo); + + var lessThanConstraint = new Constraint( + (x) => x[0], 2, 3.0, ConstraintType.LesserThanOrEqualTo); + + var greaterThanConstraint = new Constraint( + (x) => x[1], 2, 0.5, ConstraintType.GreaterThanOrEqualTo); + + // Inner solver + var initial = new double[] { 1, 3 }; + var lower = new double[] { -10, -10 }; + var upper = new double[] { 10, 10 }; + var innerSolver = new BFGS(func, 2, initial, lower, upper); + + // Solve with all three constraint types + var constraints = new IConstraint[] { equalityConstraint, lessThanConstraint, greaterThanConstraint }; + var solver = new AugmentedLagrange(func, innerSolver, constraints); + solver.Minimize(); + + // Solution should be (2, 2) + Assert.AreEqual(2.0, solver.BestParameterSet.Values[0], 0.1); + Assert.AreEqual(2.0, solver.BestParameterSet.Values[1], 0.1); + // Objective = 4+4 = 8 + Assert.AreEqual(8.0, solver.BestParameterSet.Fitness, 0.5); + } + + /// + /// Tests AugmentedLagrange with mixed constraints where the inequality constraints are binding. + /// + /// + /// Minimize (x-5)² + (y-5)² subject to: + /// x + y = 4 (equality, binding) + /// x ≤ 1 (lesser-than, binding) + /// y ≥ 2 (greater-than, not binding since y=3) + /// + /// Solution: x=1, y=3 (x is capped at 1 by the inequality, y=4-1=3) + /// + [TestMethod] + public void Test_MixedConstraints_Binding() + { + Func func = (double[] x) => + { + return Math.Pow(x[0] - 5, 2) + Math.Pow(x[1] - 5, 2); + }; + + var equalityConstraint = new Constraint( + (x) => x[0] + x[1], 2, 4.0, ConstraintType.EqualTo); + + var lessThanConstraint = new Constraint( + (x) => x[0], 2, 1.0, ConstraintType.LesserThanOrEqualTo); + + var greaterThanConstraint = new Constraint( + (x) => x[1], 2, 2.0, ConstraintType.GreaterThanOrEqualTo); + + var initial = new double[] { 0.5, 3.5 }; + var lower = new double[] { -10, -10 }; + var upper = new double[] { 10, 10 }; + var innerSolver = new BFGS(func, 2, initial, lower, upper); + + var constraints = new IConstraint[] { equalityConstraint, lessThanConstraint, greaterThanConstraint }; + var solver = new AugmentedLagrange(func, innerSolver, constraints); + solver.Minimize(); + + // Solution should be (1, 3) + Assert.AreEqual(1.0, solver.BestParameterSet.Values[0], 0.1); + Assert.AreEqual(3.0, solver.BestParameterSet.Values[1], 0.1); + } } } diff --git a/Test_Numerics/Mathematics/Special Functions/Test_Bessel.cs b/Test_Numerics/Mathematics/Special Functions/Test_Bessel.cs new file mode 100644 index 00000000..6b7ce4ac --- /dev/null +++ b/Test_Numerics/Mathematics/Special Functions/Test_Bessel.cs @@ -0,0 +1,478 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Mathematics.SpecialFunctions; + +namespace Mathematics.SpecialFunctions +{ + /// + /// Unit tests for the Bessel functions class. + /// + /// + /// + /// Authors: + /// + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + /// References: + /// + /// + /// Reference values computed using Wolfram Alpha and Abramowitz & Stegun tables. + /// + /// + [TestClass] + public class Test_Bessel + { + + #region Modified Bessel Functions of the First Kind (I) + + /// + /// Test I0 at known values. + /// + [TestMethod] + public void Test_I0() + { + // I0(0) = 1 + Assert.AreEqual(1.0, Bessel.I0(0), 1e-7); + // I0(1) ≈ 1.2660658778 + Assert.AreEqual(1.2660658778, Bessel.I0(1), 1e-5); + // I0(2) ≈ 2.2795853024 + Assert.AreEqual(2.2795853024, Bessel.I0(2), 1e-5); + // I0(5) ≈ 27.2398718236 + Assert.AreEqual(27.2398718236, Bessel.I0(5), 1e-2); + // I0(10) ≈ 2815.7166284 + Assert.AreEqual(2815.7166284, Bessel.I0(10), 1e0); + // I0 is even: I0(-x) = I0(x) + Assert.AreEqual(Bessel.I0(3.5), Bessel.I0(-3.5), 1e-10); + } + + /// + /// Test I1 at known values. + /// + [TestMethod] + public void Test_I1() + { + // I1(0) = 0 + Assert.AreEqual(0.0, Bessel.I1(0), 1e-7); + // I1(1) ≈ 0.5651591040 + Assert.AreEqual(0.5651591040, Bessel.I1(1), 1e-5); + // I1(2) ≈ 1.5906368546 + Assert.AreEqual(1.5906368546, Bessel.I1(2), 1e-5); + // I1(5) ≈ 24.3356421088 + Assert.AreEqual(24.3356421088, Bessel.I1(5), 1e-2); + // I1 is odd: I1(-x) = -I1(x) + Assert.AreEqual(-Bessel.I1(2.5), Bessel.I1(-2.5), 1e-10); + } + + /// + /// Test In for arbitrary integer order. + /// + [TestMethod] + public void Test_In() + { + // In(0, x) = I0(x) + Assert.AreEqual(Bessel.I0(2), Bessel.In(0, 2), 1e-7); + // In(1, x) = I1(x) + Assert.AreEqual(Bessel.I1(2), Bessel.In(1, 2), 1e-7); + // I2(1) ≈ 0.1357476698 + Assert.AreEqual(0.1357476698, Bessel.In(2, 1), 1e-5); + // I3(2) ≈ 0.2127399592 + Assert.AreEqual(0.2127399592, Bessel.In(3, 2), 1e-5); + // I5(3) ≈ 0.0912064773 + Assert.AreEqual(0.0912064773, Bessel.In(5, 3), 1e-5); + // In(n, 0) = 0 for n > 0 + Assert.AreEqual(0.0, Bessel.In(5, 0), 1e-10); + // Parity: In(n, -x) = (-1)^n * In(n, x) + Assert.AreEqual(Bessel.In(2, 3.0), Bessel.In(2, -3.0), 1e-7); // even n + Assert.AreEqual(-Bessel.In(3, 3.0), Bessel.In(3, -3.0), 1e-7); // odd n + } + + /// + /// Test that In throws for negative order. + /// + [TestMethod] + public void Test_In_NegativeOrder() + { + Assert.Throws(() => Bessel.In(-1, 1.0)); + } + + #endregion + + #region Modified Bessel Functions of the Second Kind (K) + + /// + /// Test K0 at known values. + /// + [TestMethod] + public void Test_K0() + { + // K0(1) ≈ 0.4210244382 + Assert.AreEqual(0.4210244382, Bessel.K0(1), 1e-5); + // K0(2) ≈ 0.1138938727 + Assert.AreEqual(0.1138938727, Bessel.K0(2), 1e-5); + // K0(5) ≈ 0.003691098334 + Assert.AreEqual(0.003691098334, Bessel.K0(5), 1e-6); + // K0(0.1) ≈ 2.4270690248 + Assert.AreEqual(2.4270690248, Bessel.K0(0.1), 1e-4); + // K0 is monotonically decreasing + Assert.IsGreaterThan(Bessel.K0(2), Bessel.K0(1)); + Assert.IsGreaterThan(Bessel.K0(5), Bessel.K0(2)); + } + + /// + /// Test K1 at known values. + /// + [TestMethod] + public void Test_K1() + { + // K1(1) ≈ 0.6019072302 + Assert.AreEqual(0.6019072302, Bessel.K1(1), 1e-5); + // K1(2) ≈ 0.1398658818 + Assert.AreEqual(0.1398658818, Bessel.K1(2), 1e-5); + // K1(5) ≈ 0.004044613184 + Assert.AreEqual(0.004044613184, Bessel.K1(5), 1e-6); + // K1 is monotonically decreasing for x > 0 + Assert.IsGreaterThan(Bessel.K1(2), Bessel.K1(1)); + } + + /// + /// Test Kn for arbitrary integer order. + /// + [TestMethod] + public void Test_Kn() + { + // Kn(0, x) = K0(x) + Assert.AreEqual(Bessel.K0(2), Bessel.Kn(0, 2), 1e-7); + // Kn(1, x) = K1(x) + Assert.AreEqual(Bessel.K1(2), Bessel.Kn(1, 2), 1e-7); + // K2(1) ≈ 1.6248388986 + Assert.AreEqual(1.6248388986, Bessel.Kn(2, 1), 1e-4); + // K3(2) ≈ 0.6473853909 + Assert.AreEqual(0.6473853909, Bessel.Kn(3, 2), 1e-4); + // Kn increases with n for fixed x + Assert.IsGreaterThan(Bessel.Kn(2, 2), Bessel.Kn(3, 2)); + } + + /// + /// Test that K functions throw for non-positive argument. + /// + [TestMethod] + public void Test_K_InvalidArgument() + { + Assert.Throws(() => Bessel.K0(0)); + Assert.Throws(() => Bessel.K0(-1)); + Assert.Throws(() => Bessel.K1(0)); + Assert.Throws(() => Bessel.K1(-1)); + Assert.Throws(() => Bessel.Kn(2, 0)); + Assert.Throws(() => Bessel.Kn(2, -1)); + } + + #endregion + + #region Bessel Functions of the First Kind (J) + + /// + /// Test J0 at known values. + /// + [TestMethod] + public void Test_J0() + { + // J0(0) = 1 + Assert.AreEqual(1.0, Bessel.J0(0), 1e-7); + // J0(1) ≈ 0.7651976866 + Assert.AreEqual(0.7651976866, Bessel.J0(1), 1e-7); + // J0(2) ≈ 0.2238907791 + Assert.AreEqual(0.2238907791, Bessel.J0(2), 1e-7); + // J0(5) ≈ -0.1775967713 + Assert.AreEqual(-0.1775967713, Bessel.J0(5), 1e-7); + // J0(10) ≈ -0.2459357645 + Assert.AreEqual(-0.2459357645, Bessel.J0(10), 1e-7); + // J0(20) ≈ 0.1670246643 + Assert.AreEqual(0.1670246643, Bessel.J0(20), 1e-6); + // J0 is even: J0(-x) = J0(x) + Assert.AreEqual(Bessel.J0(3.0), Bessel.J0(-3.0), 1e-10); + } + + /// + /// Test J1 at known values. + /// + [TestMethod] + public void Test_J1() + { + // J1(0) = 0 + Assert.AreEqual(0.0, Bessel.J1(0), 1e-7); + // J1(1) ≈ 0.4400505857 + Assert.AreEqual(0.4400505857, Bessel.J1(1), 1e-7); + // J1(2) ≈ 0.5767248078 + Assert.AreEqual(0.5767248078, Bessel.J1(2), 1e-7); + // J1(5) ≈ -0.3275791376 + Assert.AreEqual(-0.3275791376, Bessel.J1(5), 1e-7); + // J1(10) ≈ 0.0434727462 + Assert.AreEqual(0.0434727462, Bessel.J1(10), 1e-7); + // J1 is odd: J1(-x) = -J1(x) + Assert.AreEqual(-Bessel.J1(3.0), Bessel.J1(-3.0), 1e-10); + } + + /// + /// Test Jn for arbitrary integer order. + /// + [TestMethod] + public void Test_Jn() + { + // Jn(0, x) = J0(x) + Assert.AreEqual(Bessel.J0(2), Bessel.Jn(0, 2), 1e-7); + // Jn(1, x) = J1(x) + Assert.AreEqual(Bessel.J1(2), Bessel.Jn(1, 2), 1e-7); + // J2(1) ≈ 0.1149034849 + Assert.AreEqual(0.1149034849, Bessel.Jn(2, 1), 1e-7); + // J3(2) ≈ 0.1289432494 + Assert.AreEqual(0.1289432494, Bessel.Jn(3, 2), 1e-7); + // J5(5) ≈ 0.2611405461 + Assert.AreEqual(0.2611405461, Bessel.Jn(5, 5), 1e-6); + // Jn(n, 0) = 0 for n > 0 + Assert.AreEqual(0.0, Bessel.Jn(5, 0), 1e-10); + // J10(10) ≈ 0.2074861066 + Assert.AreEqual(0.2074861066, Bessel.Jn(10, 10), 1e-5); + // Parity: Jn(n, -x) = (-1)^n * Jn(n, x) + Assert.AreEqual(Bessel.Jn(2, 3.0), Bessel.Jn(2, -3.0), 1e-7); // even n + Assert.AreEqual(-Bessel.Jn(3, 3.0), Bessel.Jn(3, -3.0), 1e-7); // odd n + } + + /// + /// Test Jn using the forward recurrence path (x > n). + /// + [TestMethod] + public void Test_Jn_ForwardRecurrence() + { + // J2(10) ≈ 0.2546303137 — uses forward recurrence since |x| > n + Assert.AreEqual(0.2546303137, Bessel.Jn(2, 10), 1e-6); + // J3(10) ≈ 0.0583793794 + Assert.AreEqual(0.0583793794, Bessel.Jn(3, 10), 1e-6); + } + + /// + /// Test Jn using the Miller's downward recurrence path (x <= n). + /// + [TestMethod] + public void Test_Jn_MillerRecurrence() + { + // J10(5) ≈ 0.0014678027 — uses Miller's since |x| <= n + Assert.AreEqual(0.0014678027, Bessel.Jn(10, 5), 1e-7); + // J20(10) ≈ 1.15134e-5 (very small, deep into Miller's regime) + Assert.AreEqual(1.1513369e-5, Bessel.Jn(20, 10), 1e-9); + } + + /// + /// Test that Jn throws for negative order. + /// + [TestMethod] + public void Test_Jn_NegativeOrder() + { + Assert.Throws(() => Bessel.Jn(-1, 1.0)); + } + + #endregion + + #region Bessel Functions of the Second Kind (Y) + + /// + /// Test Y0 at known values. + /// + [TestMethod] + public void Test_Y0() + { + // Y0(1) ≈ 0.0882569642 + Assert.AreEqual(0.0882569642, Bessel.Y0(1), 1e-7); + // Y0(2) ≈ 0.5103756726 + Assert.AreEqual(0.5103756726, Bessel.Y0(2), 1e-7); + // Y0(5) ≈ -0.3085176252 + Assert.AreEqual(-0.3085176252, Bessel.Y0(5), 1e-7); + // Y0(10) ≈ 0.0556711673 + Assert.AreEqual(0.0556711673, Bessel.Y0(10), 1e-7); + // Y0(20) ≈ 0.0626405967 + Assert.AreEqual(0.0626405967, Bessel.Y0(20), 1e-6); + } + + /// + /// Test Y1 at known values. + /// + [TestMethod] + public void Test_Y1() + { + // Y1(1) ≈ -0.7812128213 + Assert.AreEqual(-0.7812128213, Bessel.Y1(1), 1e-7); + // Y1(2) ≈ -0.1070324315 + Assert.AreEqual(-0.1070324315, Bessel.Y1(2), 1e-7); + // Y1(5) ≈ 0.1478631434 + Assert.AreEqual(0.1478631434, Bessel.Y1(5), 1e-7); + // Y1(10) ≈ 0.2490154242 + Assert.AreEqual(0.2490154242, Bessel.Y1(10), 1e-7); + } + + /// + /// Test Yn for arbitrary integer order. + /// + [TestMethod] + public void Test_Yn() + { + // Yn(0, x) = Y0(x) + Assert.AreEqual(Bessel.Y0(2), Bessel.Yn(0, 2), 1e-7); + // Yn(1, x) = Y1(x) + Assert.AreEqual(Bessel.Y1(2), Bessel.Yn(1, 2), 1e-7); + // Y2(1) ≈ -1.6506826068 + Assert.AreEqual(-1.6506826068, Bessel.Yn(2, 1), 1e-5); + // Y3(2) ≈ -1.1277837769 + Assert.AreEqual(-1.1277837769, Bessel.Yn(3, 2), 1e-5); + // Y5(5) ≈ -0.4536948225 + Assert.AreEqual(-0.4536948225, Bessel.Yn(5, 5), 1e-5); + } + + /// + /// Test that Y functions throw for non-positive argument. + /// + [TestMethod] + public void Test_Y_InvalidArgument() + { + Assert.Throws(() => Bessel.Y0(0)); + Assert.Throws(() => Bessel.Y0(-1)); + Assert.Throws(() => Bessel.Y1(0)); + Assert.Throws(() => Bessel.Y1(-1)); + Assert.Throws(() => Bessel.Yn(2, 0)); + Assert.Throws(() => Bessel.Yn(2, -1)); + } + + #endregion + + #region Cross-Function Identities + + /// + /// Verify the Wronskian identity: J0(x)Y1(x) - J1(x)Y0(x) = -2/(pi*x). + /// + [TestMethod] + public void Test_Wronskian_J0Y1_J1Y0() + { + double[] testValues = { 0.5, 1.0, 2.0, 5.0, 10.0 }; + foreach (double x in testValues) + { + double wronskian = Bessel.J0(x) * Bessel.Y1(x) - Bessel.J1(x) * Bessel.Y0(x); + double expected = -2.0 / (Math.PI * x); + Assert.AreEqual(expected, wronskian, 1e-6, $"Wronskian failed at x={x}"); + } + } + + /// + /// Verify the modified Wronskian identity: I0(x)K1(x) + I1(x)K0(x) = 1/x. + /// + [TestMethod] + public void Test_Wronskian_I0K1_I1K0() + { + double[] testValues = { 0.5, 1.0, 2.0, 5.0, 10.0 }; + foreach (double x in testValues) + { + double wronskian = Bessel.I0(x) * Bessel.K1(x) + Bessel.I1(x) * Bessel.K0(x); + double expected = 1.0 / x; + Assert.AreEqual(expected, wronskian, 1e-5, $"Modified Wronskian failed at x={x}"); + } + } + + /// + /// Verify that I0'(x) = I1(x) numerically using finite differences. + /// + [TestMethod] + public void Test_I0_Derivative_Equals_I1() + { + double[] testValues = { 0.5, 1.0, 2.0, 5.0 }; + double h = 1e-6; + foreach (double x in testValues) + { + double numericalDerivative = (Bessel.I0(x + h) - Bessel.I0(x - h)) / (2.0 * h); + Assert.AreEqual(Bessel.I1(x), numericalDerivative, 1e-4, $"I0'(x) != I1(x) at x={x}"); + } + } + + /// + /// Verify the addition formula: J0(x) + 2*J2(x) + 2*J4(x) + 2*J6(x) + ... = 1. + /// + [TestMethod] + public void Test_J_SumIdentity() + { + double x = 3.0; + double sum = Bessel.J0(x); + for (int k = 1; k <= 20; k++) + { + sum += 2.0 * Bessel.Jn(2 * k, x); + } + Assert.AreEqual(1.0, sum, 1e-6); + } + + #endregion + + #region Large Argument Tests + + /// + /// Test J0 for large arguments where the asymptotic expansion is used. + /// + [TestMethod] + public void Test_J0_LargeArgument() + { + // J0(50) ≈ 0.0558123276 + Assert.AreEqual(0.0558123276, Bessel.J0(50), 1e-5); + // J0(100) ≈ 0.0199858503 + Assert.AreEqual(0.0199858503, Bessel.J0(100), 1e-4); + } + + /// + /// Test I0 for large arguments where the asymptotic expansion is used. + /// + [TestMethod] + public void Test_I0_LargeArgument() + { + // I0(20) ≈ 4.355828256e7 + Assert.AreEqual(4.355828256e7, Bessel.I0(20), 1e3); + } + + /// + /// Test K0 for moderate arguments. + /// + [TestMethod] + public void Test_K0_ModerateArgument() + { + // K0(10) ≈ 1.778006232e-5 + Assert.AreEqual(1.778006232e-5, Bessel.K0(10), 1e-8); + } + + #endregion + + } +} diff --git a/Test_Numerics/Sampling/MCMC/Test_HMC.cs b/Test_Numerics/Sampling/MCMC/Test_HMC.cs index 3f7cca47..a7a16af5 100644 --- a/Test_Numerics/Sampling/MCMC/Test_HMC.cs +++ b/Test_Numerics/Sampling/MCMC/Test_HMC.cs @@ -105,5 +105,42 @@ double logLH(double[] x) Assert.AreEqual(5771.81, results.ParameterResults[1].SummaryStatistics.UpperCI, 0.05 * 5771.81); } + /// + /// Verifies that HMC does not crash when the gradient function encounters non-finite values. + /// This can happen when leapfrog integration drifts parameters into regions where the + /// log-likelihood returns -Infinity, causing NumericalDerivative.Gradient to throw. + /// + [TestMethod] + public void Test_HMC_NonFiniteGradient_DoesNotCrash() + { + // Use narrow priors that make it easy for leapfrog to drift into invalid regions + var muPrior = new Uniform(-100, 100); + var sigmaPrior = new Uniform(0.01, 50); + var priors = new List { muPrior, sigmaPrior }; + + // Simple data + double[] data = { 10.0, 12.0, 11.0, 13.0, 9.0, 14.0, 10.5, 11.5 }; + + // Log-likelihood that throws for invalid sigma (sigma <= 0) + double logLH(double[] x) + { + var dist = new Normal(x[0], x[1]); + return dist.LogLikelihood(data); + } + + // Use a large step size and many steps to increase chance of drifting out of bounds + var sampler = new HMC(priors, logLH, stepSize: 1.0, steps: 20); + sampler.NumberOfChains = 2; + sampler.WarmupIterations = 100; + sampler.Iterations = 200; + + // This should complete without throwing AggregateException/ArithmeticException + sampler.Sample(); + + // Verify we got some results + Assert.IsNotNull(sampler.MarkovChains); + Assert.IsNotEmpty(sampler.MarkovChains, "Expected at least one Markov chain"); + } + } } diff --git a/Test_Numerics/Sampling/MCMC/Test_MCMCDiagnostics.cs b/Test_Numerics/Sampling/MCMC/Test_MCMCDiagnostics.cs new file mode 100644 index 00000000..74d01597 --- /dev/null +++ b/Test_Numerics/Sampling/MCMC/Test_MCMCDiagnostics.cs @@ -0,0 +1,105 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System.Collections.Generic; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Mathematics.Optimization; +using Numerics.Sampling.MCMC; + +namespace Sampling.MCMC +{ + /// + /// Unit tests for MCMCDiagnostics, particularly GelmanRubin R-hat with warmup. + /// + [TestClass] + public class Test_MCMCDiagnostics + { + /// + /// Verify Gelman-Rubin R-hat computation with warmup correctly divides by + /// (N - warmupIterations) instead of N. For chains from the same distribution, + /// R-hat should be close to 1.0. + /// + [TestMethod] + public void Test_GelmanRubin_WithWarmup() + { + // Create 3 chains of 200 samples each from similar distributions. + // Use deterministic sequences for reproducibility. + var rng1 = new Random(42); + var rng2 = new Random(123); + var rng3 = new Random(456); + int chainLength = 200; + + var chain1 = new List(); + var chain2 = new List(); + var chain3 = new List(); + + for (int i = 0; i < chainLength; i++) + { + // Simulate chains that start dispersed but converge + double drift1 = i < 50 ? 5.0 : 0.0; + double drift2 = i < 50 ? -5.0 : 0.0; + double drift3 = i < 50 ? 3.0 : 0.0; + + chain1.Add(new ParameterSet(new[] { rng1.NextDouble() * 2 - 1 + drift1 }, 0)); + chain2.Add(new ParameterSet(new[] { rng2.NextDouble() * 2 - 1 + drift2 }, 0)); + chain3.Add(new ParameterSet(new[] { rng3.NextDouble() * 2 - 1 + drift3 }, 0)); + } + + var chains = new List> { chain1, chain2, chain3 }; + + // Without warmup, R-hat should be high (chains have different initial distributions) + var rhatNoWarmup = MCMCDiagnostics.GelmanRubin(chains, 0); + Assert.IsGreaterThan(1.1, rhatNoWarmup[0], $"R-hat without warmup should be > 1.1, got {rhatNoWarmup[0]}"); + + // With warmup=50, R-hat should be close to 1.0 (converged portion only) + var rhatWithWarmup = MCMCDiagnostics.GelmanRubin(chains, 50); + Assert.IsLessThan(1.1, rhatWithWarmup[0], $"R-hat with warmup=50 should be < 1.1, got {rhatWithWarmup[0]}"); + + // Warmup should improve R-hat (make it closer to 1.0) + Assert.IsLessThan(rhatNoWarmup[0], rhatWithWarmup[0], + "R-hat with warmup should be closer to 1.0 than without warmup"); + } + + /// + /// Verify GelmanRubin handles edge cases. + /// + [TestMethod] + public void Test_GelmanRubin_EdgeCases() + { + // Single chain should return NaN + var chain = new List(); + for (int i = 0; i < 10; i++) + chain.Add(new ParameterSet(new[] { 1.0 }, 0)); + var singleChain = new List> { chain }; + var result = MCMCDiagnostics.GelmanRubin(singleChain); + Assert.IsTrue(double.IsNaN(result[0]), "Single chain should return NaN"); + } + } +} diff --git a/Test_Numerics/Sampling/MCMC/Test_NUTS.cs b/Test_Numerics/Sampling/MCMC/Test_NUTS.cs new file mode 100644 index 00000000..f6d5abc6 --- /dev/null +++ b/Test_Numerics/Sampling/MCMC/Test_NUTS.cs @@ -0,0 +1,221 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using System.Collections.Generic; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics.Distributions; +using Numerics.Sampling.MCMC; + +namespace Sampling.MCMC +{ + /// + /// Unit tests for the No-U-Turn Sampler (NUTS). + /// + /// + /// + /// Authors: + /// + /// Haden Smith, USACE Risk Management Center, cole.h.smith@usace.army.mil + /// + /// + /// + /// References: + /// + /// + /// Reference values verified against RStan NUTS results for the Tippecanoe River dataset. + /// + /// + [TestClass] + public class Test_NUTS + { + + // Reference: "Flood Frequency Analysis", A.R. Rao & K.H. Hamed, CRC Press, 2000. + // Table 5.1.1 Tippecanoe River Near Delphi, Indiana (Station 43) Data + private double[] sample1 = new double[] { 6290d, 2700d, 13100d, 16900d, 14600d, 9600d, 7740d, 8490d, 8130d, 12000d, 17200d, 15000d, 12400d, 6960d, 6500d, 5840d, 10400d, 18800d, 21400d, 22600d, 14200d, 11000d, 12800d, 15700d, 4740d, 6950d, 11800d, 12100d, 20600d, 14600d, 14600d, 8900d, 10600d, 14200d, 14100d, 14100d, 12500d, 7530d, 13400d, 17600d, 13400d, 19200d, 16900d, 15500d, 14500d, 21900d, 10400d, 7460d }; + + // Reference: "Flood Frequency Analysis", A.R. Rao & K.H. Hamed, CRC Press, 2000. + // Table 7.2.1 Sugar Creek at Crawfordsville, IN + private double[] sample2 = new double[] { 17600d, 3660d, 903d, 5050d, 24000d, 11400d, 9470d, 8970d, 7710d, 14800d, 13900d, 20800d, 9470d, 7860d, 7860d, 2730d, 6480d, 18200d, 26300d, 15100d, 14600d, 7300d, 8580d, 15100d, 15100d, 21800d, 6200d, 2130d, 11100d, 14300d, 11200d, 6670d, 5440d, 9370d, 6900d, 9680d, 6810d, 7730d, 5290d, 12200d, 9750d, 7390d, 13100d, 7190d, 8850d, 6290d, 18800d, 9740d, 2990d, 6950d, 9390d, 12400d, 21200d }; + + /// + /// This test compares the results obtained using NUTS for the Normal distribution with those from the 'rstan' package. + /// + [TestMethod] + public void Test_NUTS_NormalDist_RStan() + { + + // Create uniform priors + var normDist = new Normal(); + var constraints = normDist.GetParameterConstraints(sample1); + var muPrior = new Uniform(constraints.Item2[0], constraints.Item3[0]); + var sigmaPrior = new Uniform(constraints.Item2[1], constraints.Item3[1]); + var priors = new List { muPrior, sigmaPrior }; + + // Create log-likelihood function + double logLH(double[] x) + { + var dist = new Normal(x[0], x[1]); + return dist.LogLikelihood(sample1); + } + + // Create and run MCMC sampler + var sampler = new NUTS(priors, logLH); + sampler.Sample(); + var results = new MCMCResults(sampler); + + /* Below are the results from 'rstan' using comparable MCMC settings: + * mean se_mean sd 5% 50% 95% n_eff Rhat + * mu 12663.69 7.10 706.60 11488.50 12671.08 13801.45 9897 1 + * sigma 4844.09 5.22 519.08 4077.80 4796.63 5771.81 9880 1 + * lp__ -466.13 0.01 1.03 -468.17 -465.81 -465.15 9958 1 + * + * Since MCMC methods rely on random number generation, results will not be + * exactly the same as those produced by other samplers. Therefore, these + * comparisons aim to verify whether the results are within 5% of 'rstan' results. + */ + + // Mu + Assert.AreEqual(12663.69, results.ParameterResults[0].SummaryStatistics.Mean, 0.05 * 12663.69); + Assert.AreEqual(706.60, results.ParameterResults[0].SummaryStatistics.StandardDeviation, 0.05 * 706.60); + Assert.AreEqual(11488.50, results.ParameterResults[0].SummaryStatistics.LowerCI, 0.05 * 11488.50); + Assert.AreEqual(12671.08, results.ParameterResults[0].SummaryStatistics.Median, 0.05 * 12671.08); + Assert.AreEqual(13801.45, results.ParameterResults[0].SummaryStatistics.UpperCI, 0.05 * 13801.45); + // Sigma + Assert.AreEqual(4844.09, results.ParameterResults[1].SummaryStatistics.Mean, 0.05 * 4844.09); + Assert.AreEqual(519.08, results.ParameterResults[1].SummaryStatistics.StandardDeviation, 0.05 * 519.08); + Assert.AreEqual(4077.80, results.ParameterResults[1].SummaryStatistics.LowerCI, 0.05 * 4077.80); + Assert.AreEqual(4796.63, results.ParameterResults[1].SummaryStatistics.Median, 0.05 * 4796.63); + Assert.AreEqual(5771.81, results.ParameterResults[1].SummaryStatistics.UpperCI, 0.05 * 5771.81); + } + + /// + /// This test compares the results obtained using NUTS for the Logistic distribution with those from the 'rstan' package. + /// + [TestMethod] + public void Test_NUTS_LogisticDist_RStan() + { + + // Create uniform priors + var logDist = new Logistic(); + var constraints = logDist.GetParameterConstraints(sample1); + var xiPrior = new Uniform(constraints.Item2[0], constraints.Item3[0]); + var alphaPrior = new Uniform(constraints.Item2[1], constraints.Item3[1]); + var priors = new List { xiPrior, alphaPrior }; + + // Create log-likelihood function + double logLH(double[] x) + { + var dist = new Logistic(x[0], x[1]); + return dist.LogLikelihood(sample1); + } + + // Create and run sampler + var sampler = new NUTS(priors, logLH); + sampler.Sample(); + var results = new MCMCResults(sampler); + + /* Below are the results from 'rstan' using comparable MCMC settings: + * mean se_mean sd 5% 50% 95% n_eff Rhat + * mu 12631.74 7.19 719.36 11458.52 12622.01 13830.74 10002 1 + * sigma 2823.47 3.49 348.26 2307.77 2794.40 3441.31 9947 1 + * lp__ -467.66 0.01 1.05 -469.75 -467.33 -466.69 10128 + * + * Since MCMC methods rely on random number generation, results will not be + * exactly the same as those produced by other samplers. Therefore, these + * comparisons aim to verify whether the results are within 5% of 'rstan' results. + */ + + // Mu in R is equal to Xi in Numerics + Assert.AreEqual(12631.74, results.ParameterResults[0].SummaryStatistics.Mean, 0.05 * 12631.74); + Assert.AreEqual(719.36, results.ParameterResults[0].SummaryStatistics.StandardDeviation, 0.05 * 719.36); + Assert.AreEqual(11458.52, results.ParameterResults[0].SummaryStatistics.LowerCI, 0.05 * 11458.52); + Assert.AreEqual(12622.01, results.ParameterResults[0].SummaryStatistics.Median, 0.05 * 12622.01); + Assert.AreEqual(13830.74, results.ParameterResults[0].SummaryStatistics.UpperCI, 0.05 * 13830.74); + // Sigma in R is equal to Alpha in Numerics + Assert.AreEqual(2823.47, results.ParameterResults[1].SummaryStatistics.Mean, 0.05 * 2823.47); + Assert.AreEqual(348.26, results.ParameterResults[1].SummaryStatistics.StandardDeviation, 0.05 * 348.26); + Assert.AreEqual(2307.77, results.ParameterResults[1].SummaryStatistics.LowerCI, 0.05 * 2307.77); + Assert.AreEqual(2794.40, results.ParameterResults[1].SummaryStatistics.Median, 0.05 * 2794.40); + Assert.AreEqual(3441.31, results.ParameterResults[1].SummaryStatistics.UpperCI, 0.05 * 3441.31); + } + + /// + /// This test compares the results obtained using NUTS for the Gumbel distribution with those from the 'rstan' package. + /// + [TestMethod] + public void Test_NUTS_GumbelDist_RStan() + { + + // Create uniform priors + var gumDist = new Gumbel(); + var constraints = gumDist.GetParameterConstraints(sample2); + var xiPrior = new Uniform(constraints.Item2[0], constraints.Item3[0]); + var alphaPrior = new Uniform(constraints.Item2[1], constraints.Item3[1]); + var priors = new List { xiPrior, alphaPrior }; + + // Create log-likelihood function + double logLH(double[] x) + { + var dist = new Gumbel(x[0], x[1]); + return dist.LogLikelihood(sample2); + } + + // Create and run sampler + var sampler = new NUTS(priors, logLH); + sampler.Sample(); + var results = new MCMCResults(sampler); + + /* Below are the results from 'rstan' using comparable MCMC settings: + * mean se_mean sd 5% 50% 95% n_eff Rhat + * mu 8067.90 6.78 680.85 6973.92 8055.82 9201.14 10095 1 + * beta 4658.04 5.32 522.33 3885.53 4612.71 5595.40 9623 1 + * lp__ -521.82 0.01 1.02 -523.81 -521.50 -520.84 9860 1 + * + * Since MCMC methods rely on random number generation, results will not be + * exactly the same as those produced by other samplers. Therefore, these + * comparisons aim to verify whether the results are within 5% of 'rstan' results. + */ + + // Mu in R is equal to Xi in Numerics + Assert.AreEqual(8067.90, results.ParameterResults[0].SummaryStatistics.Mean, 0.05 * 8067.90); + Assert.AreEqual(680.85, results.ParameterResults[0].SummaryStatistics.StandardDeviation, 0.05 * 680.85); + Assert.AreEqual(6973.92, results.ParameterResults[0].SummaryStatistics.LowerCI, 0.05 * 6973.92); + Assert.AreEqual(8055.82, results.ParameterResults[0].SummaryStatistics.Median, 0.05 * 8055.82); + Assert.AreEqual(9201.14, results.ParameterResults[0].SummaryStatistics.UpperCI, 0.05 * 9201.14); + // Beta in R is equal to Alpha in Numerics + Assert.AreEqual(4658.04, results.ParameterResults[1].SummaryStatistics.Mean, 0.05 * 4658.04); + Assert.AreEqual(522.33, results.ParameterResults[1].SummaryStatistics.StandardDeviation, 0.05 * 522.33); + Assert.AreEqual(3885.53, results.ParameterResults[1].SummaryStatistics.LowerCI, 0.05 * 3885.53); + Assert.AreEqual(4612.71, results.ParameterResults[1].SummaryStatistics.Median, 0.05 * 4612.71); + Assert.AreEqual(5595.40, results.ParameterResults[1].SummaryStatistics.UpperCI, 0.05 * 5595.40); + } + + } +} diff --git a/Test_Numerics/Sampling/Test_Bootstrap.cs b/Test_Numerics/Sampling/Test_Bootstrap.cs new file mode 100644 index 00000000..087baea2 --- /dev/null +++ b/Test_Numerics/Sampling/Test_Bootstrap.cs @@ -0,0 +1,447 @@ +/* +* NOTICE: +* The U.S. Army Corps of Engineers, Risk Management Center (USACE-RMC) makes no guarantees about +* the results, or appropriateness of outputs, obtained from Numerics. +* +* LIST OF CONDITIONS: +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* ● Redistributions of source code must retain the above notice, this list of conditions, and the +* following disclaimer. +* ● Redistributions in binary form must reproduce the above notice, this list of conditions, and +* the following disclaimer in the documentation and/or other materials provided with the distribution. +* ● The names of the U.S. Government, the U.S. Army Corps of Engineers, the Institute for Water +* Resources, or the Risk Management Center may not be used to endorse or promote products derived +* from this software without specific prior written permission. Nor may the names of its contributors +* be used to endorse or promote products derived from this software without specific prior +* written permission. +* +* DISCLAIMER: +* THIS SOFTWARE IS PROVIDED BY THE U.S. ARMY CORPS OF ENGINEERS RISK MANAGEMENT CENTER +* (USACE-RMC) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL USACE-RMC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Numerics; +using Numerics.Data.Statistics; +using Numerics.Distributions; +using Numerics.Mathematics.Optimization; +using Numerics.Sampling; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace Sampling +{ + + /// + /// Unit tests for the general-purpose Bootstrap class. + /// Tests compare against the 'boot' R package reference values and Monte Carlo CIs, + /// following the same validation approach as Test_BootstrapAnalysis. + /// + [TestClass] + public class Test_Bootstrap + { + + private double _mu = 3.122599; + private double _sigma = 0.5573654; + private int _sampleSize = 100; + private double[] _probabilities = new double[] { 0.999, 0.99, 0.95, 0.9, 0.5, 0.1, 0.05, 0.01 }; + + /// + /// Creates a generic bootstrap configured to replicate parametric bootstrap of a Normal distribution + /// using the method of moments. This mirrors the BootstrapAnalysis setup used in Test_BootstrapAnalysis. + /// + private Bootstrap CreateNormalBootstrap() + { + // The "original data" is not used directly for parametric bootstrap; + // resampling is driven by the parameters. We pass null. + var parms = new ParameterSet(new double[] { _mu, _sigma }, double.NaN); + var boot = new Bootstrap(null, parms); + boot.Replicates = 10000; + boot.PRNGSeed = 12345; + + // Resample: generate random values from the Normal distribution defined by the parameter set + boot.ResampleFunction = (data, ps, rng) => + { + var d = new Normal(ps.Values[0], ps.Values[1]); + return d.GenerateRandomValues(_sampleSize, rng.Next()); + }; + + // Fit: estimate Normal parameters from sample using method of moments + boot.FitFunction = (sample) => + { + var d = new Normal(); + ((IEstimation)d).Estimate(sample, ParameterEstimationMethod.MethodOfMoments); + if (!d.ParametersValid) + throw new Exception("Invalid parameters."); + return new ParameterSet(d.GetParameters, double.NaN); + }; + + // Statistic: compute quantiles at the specified probabilities + boot.StatisticFunction = (ps) => + { + var d = new Normal(ps.Values[0], ps.Values[1]); + var result = new double[_probabilities.Length]; + for (int i = 0; i < _probabilities.Length; i++) + result[i] = d.InverseCDF(_probabilities[i]); + return result; + }; + + return boot; + } + + /// + /// Test that the percentile method produces results consistent with the 'boot' R package. + /// Reference values from Test_BootstrapAnalysis.Test_PercentileCI(). + /// + [TestMethod] + public void Test_PercentileCI() + { + var boot = CreateNormalBootstrap(); + boot.Run(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.Percentile); + + // Verify we got results for all probabilities + Assert.HasCount(_probabilities.Length, results.StatisticResults); + + // Verify valid count + for (int i = 0; i < _probabilities.Length; i++) + { + Assert.IsGreaterThan(0.95 * boot.Replicates, results.StatisticResults[i].ValidCount, + $"Too many failures at probability {_probabilities[i]}"); + } + + // Compare against BootstrapAnalysis + var dist = new Normal(_mu, _sigma); + var ba = new BootstrapAnalysis(dist, ParameterEstimationMethod.MethodOfMoments, _sampleSize); + var baCIs = ba.PercentileQuantileCI(_probabilities); + + for (int i = 0; i < _probabilities.Length; i++) + { + Assert.AreEqual(baCIs[i, 0], results.StatisticResults[i].LowerCI, 0.01 * Math.Abs(baCIs[i, 0]), + $"Lower CI mismatch at p={_probabilities[i]}"); + Assert.AreEqual(baCIs[i, 1], results.StatisticResults[i].UpperCI, 0.01 * Math.Abs(baCIs[i, 1]), + $"Upper CI mismatch at p={_probabilities[i]}"); + } + } + + /// + /// Test that the Normal (standard) method produces results consistent with the 'boot' R package. + /// Reference values from Test_BootstrapAnalysis.Test_NormalCI(). + /// + [TestMethod] + public void Test_NormalCI() + { + var boot = CreateNormalBootstrap(); + boot.Run(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.Normal); + + // Compare against BootstrapAnalysis + var dist = new Normal(_mu, _sigma); + var ba = new BootstrapAnalysis(dist, ParameterEstimationMethod.MethodOfMoments, _sampleSize); + var baCIs = ba.NormalQuantileCI(_probabilities); + + for (int i = 0; i < _probabilities.Length; i++) + { + Assert.AreEqual(baCIs[i, 0], results.StatisticResults[i].LowerCI, 0.01 * Math.Abs(baCIs[i, 0]), + $"Lower CI mismatch at p={_probabilities[i]}"); + Assert.AreEqual(baCIs[i, 1], results.StatisticResults[i].UpperCI, 0.01 * Math.Abs(baCIs[i, 1]), + $"Upper CI mismatch at p={_probabilities[i]}"); + } + } + + /// + /// Test that the bias-corrected (BC) method produces results consistent with BootstrapAnalysis. + /// + [TestMethod] + public void Test_BiasCorrectedCI() + { + var boot = CreateNormalBootstrap(); + boot.Run(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.BiasCorrected); + + // Compare against BootstrapAnalysis + var dist = new Normal(_mu, _sigma); + var ba = new BootstrapAnalysis(dist, ParameterEstimationMethod.MethodOfMoments, _sampleSize); + var baCIs = ba.BiasCorrectedQuantileCI(_probabilities); + + for (int i = 0; i < _probabilities.Length; i++) + { + Assert.AreEqual(baCIs[i, 0], results.StatisticResults[i].LowerCI, 0.01 * Math.Abs(baCIs[i, 0]), + $"Lower CI mismatch at p={_probabilities[i]}"); + Assert.AreEqual(baCIs[i, 1], results.StatisticResults[i].UpperCI, 0.01 * Math.Abs(baCIs[i, 1]), + $"Upper CI mismatch at p={_probabilities[i]}"); + } + } + + /// + /// Test that the Bootstrap-t (studentized) method produces results consistent with the "true" + /// Monte Carlo confidence intervals for the Normal distribution. + /// + [TestMethod] + public void Test_BootstrapTCI() + { + var boot = CreateNormalBootstrap(); + boot.RunWithStudentizedBootstrap(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.BootstrapT); + + // Compare against the true Monte Carlo CIs (same approach as Test_BootstrapAnalysis.Test_BootstrapTCI) + var dist = new Normal(_mu, _sigma); + var trueCIs = dist.MonteCarloConfidenceIntervals(_sampleSize, 10000, _probabilities, new double[] { 0.05, 0.95 }); + + for (int i = 0; i < _probabilities.Length; i++) + { + Assert.AreEqual(trueCIs[i, 0], results.StatisticResults[i].LowerCI, 0.01 * Math.Abs(trueCIs[i, 0]), + $"Lower CI mismatch at p={_probabilities[i]}"); + Assert.AreEqual(trueCIs[i, 1], results.StatisticResults[i].UpperCI, 0.01 * Math.Abs(trueCIs[i, 1]), + $"Upper CI mismatch at p={_probabilities[i]}"); + } + } + + /// + /// Test that the BCa method produces results consistent with the "true" + /// Monte Carlo confidence intervals for the Normal distribution. + /// + [TestMethod] + public void Test_BCaCI() + { + // Use the same sample data as Test_BootstrapAnalysis.Test_BCaCI + var sampleData = new double[] { 3.292764, 3.354733, 2.945348, 2.773251, 3.302944, 2.091022, 3.315049, 2.861908, 2.85792, 2.540339, 2.941876, 3.908656, 3.185314, 3.260108, 2.624734, 3.40845, 2.556821, 2.834211, 3.560356, 3.149362, 3.389811, 3.727893, 2.677836, 2.223431, 2.201145, 3.902549, 2.759176, 3.31019, 3.306062, 2.918845, 3.405937, 4.098417, 4.024595, 3.816223, 3.127136, 3.245594, 2.837957, 2.168975, 3.883867, 3.012901, 3.564255, 1.809821, 2.469867, 3.46857, 3.427226, 3.730365, 2.293451, 3.283702, 3.291594, 2.346601, 2.729807, 3.973846, 3.026795, 3.175831, 2.664512, 3.138977, 3.345586, 3.411898, 4.072533, 1.826528, 3.074796, 2.328734, 3.276652, 3.794981, 2.70656, 2.083811, 3.44407, 3.796744, 3.258427, 2.352164, 3.027308, 2.607675, 2.475324, 4.165256, 3.701353, 3.4713, 3.413129, 2.59423, 3.238124, 3.510629, 3.322692, 3.521572, 2.847815, 4.238555, 3.48561, 3.93355, 3.336021, 2.846023, 3.268262, 3.412435, 2.518049, 2.572459, 3.943473, 2.80409, 2.509684, 3.343666, 2.747478, 4.07886, 2.700101, 2.652727 }; + + // Fit the distribution from sample data (matching BCaQuantileCI which re-estimates) + var dist = new Normal(); + ((IEstimation)dist).Estimate(sampleData, ParameterEstimationMethod.MethodOfMoments); + + var parms = new ParameterSet(dist.GetParameters, double.NaN); + var boot = new Bootstrap(sampleData, parms); + boot.Replicates = 10000; + boot.PRNGSeed = 12345; + + boot.ResampleFunction = (data, ps, rng) => + { + var d = new Normal(ps.Values[0], ps.Values[1]); + return d.GenerateRandomValues(sampleData.Length, rng.Next()); + }; + + boot.FitFunction = (sample) => + { + var d = new Normal(); + ((IEstimation)d).Estimate(sample, ParameterEstimationMethod.MethodOfMoments); + if (!d.ParametersValid) throw new Exception("Invalid parameters."); + return new ParameterSet(d.GetParameters, double.NaN); + }; + + boot.StatisticFunction = (ps) => + { + var d = new Normal(ps.Values[0], ps.Values[1]); + var result = new double[_probabilities.Length]; + for (int i = 0; i < _probabilities.Length; i++) + result[i] = d.InverseCDF(_probabilities[i]); + return result; + }; + + // Set up jackknife delegates for BCa + boot.JackknifeFunction = (data, idx) => + { + var list = new List(data); + list.RemoveAt(idx); + return list.ToArray(); + }; + boot.SampleSizeFunction = (data) => data.Length; + + boot.Run(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.BCa); + + // Compare against the true Monte Carlo CIs + var trueCIs = dist.MonteCarloConfidenceIntervals(sampleData.Length, 10000, _probabilities, new double[] { 0.05, 0.95 }); + for (int i = 0; i < _probabilities.Length; i++) + { + Assert.AreEqual(trueCIs[i, 0], results.StatisticResults[i].LowerCI, 0.01 * Math.Abs(trueCIs[i, 0]), + $"Lower CI mismatch at p={_probabilities[i]}"); + Assert.AreEqual(trueCIs[i, 1], results.StatisticResults[i].UpperCI, 0.01 * Math.Abs(trueCIs[i, 1]), + $"Upper CI mismatch at p={_probabilities[i]}"); + } + } + + /// + /// Test that parameter-level confidence intervals are computed correctly. + /// + [TestMethod] + public void Test_ParameterCIs() + { + var boot = CreateNormalBootstrap(); + boot.Run(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.Percentile); + + // Should have 2 parameter results (mu, sigma) + Assert.HasCount(2, results.ParameterResults); + + // Population estimates should match + Assert.AreEqual(_mu, results.ParameterResults[0].PopulationEstimate, 1e-10); + Assert.AreEqual(_sigma, results.ParameterResults[1].PopulationEstimate, 1e-10); + + // CIs should bracket the population values + Assert.IsLessThan(_mu, results.ParameterResults[0].LowerCI); + Assert.IsGreaterThan(_mu, results.ParameterResults[0].UpperCI); + Assert.IsLessThan(_sigma, results.ParameterResults[1].LowerCI); + Assert.IsGreaterThan(_sigma, results.ParameterResults[1].UpperCI); + + // Valid count should be high + Assert.IsGreaterThan(0.95 * boot.Replicates, results.ParameterResults[0].ValidCount); + } + + /// + /// Test that the bootstrap handles some failed replicates and tracks valid count correctly. + /// + [TestMethod] + public void Test_ErrorHandlingAndValidCount() + { + var parms = new ParameterSet(new double[] { _mu, _sigma }, double.NaN); + var boot = new Bootstrap(null, parms); + boot.Replicates = 1000; + boot.PRNGSeed = 12345; + boot.MaxRetries = 2; + + int callCount = 0; + + boot.ResampleFunction = (data, ps, rng) => + { + var d = new Normal(ps.Values[0], ps.Values[1]); + return d.GenerateRandomValues(_sampleSize, rng.Next()); + }; + + // Fit function that fails ~10% of the time + boot.FitFunction = (sample) => + { + int count = Interlocked.Increment(ref callCount); + if (count % 10 == 0) throw new Exception("Simulated failure"); + var d = new Normal(); + ((IEstimation)d).Estimate(sample, ParameterEstimationMethod.MethodOfMoments); + return new ParameterSet(d.GetParameters, double.NaN); + }; + + boot.StatisticFunction = (ps) => + { + var d = new Normal(ps.Values[0], ps.Values[1]); + return new double[] { d.InverseCDF(0.99) }; + }; + + boot.Run(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.Percentile); + + // With retries, most replicates should succeed + Assert.IsGreaterThan(0.8 * boot.Replicates, results.StatisticResults[0].ValidCount, + $"Valid count {results.StatisticResults[0].ValidCount} too low"); + + // CIs should still be reasonable (non-NaN) + Assert.IsFalse(double.IsNaN(results.StatisticResults[0].LowerCI)); + Assert.IsFalse(double.IsNaN(results.StatisticResults[0].UpperCI)); + } + + /// + /// Test that requesting BCa without JackknifeFunction throws. + /// + [TestMethod] + public void Test_BCa_RequiresJackknifeFunction() + { + var boot = CreateNormalBootstrap(); + boot.Run(); + Assert.Throws(() => boot.GetConfidenceIntervals(BootstrapCIMethod.BCa)); + } + + /// + /// Test that requesting Bootstrap-t without RunWithStudentizedBootstrap throws. + /// + [TestMethod] + public void Test_BootstrapT_RequiresStudentizedRun() + { + var boot = CreateNormalBootstrap(); + boot.Run(); + Assert.Throws(() => boot.GetConfidenceIntervals(BootstrapCIMethod.BootstrapT)); + } + + /// + /// Test that calling GetConfidenceIntervals before Run() throws. + /// + [TestMethod] + public void Test_GetCI_RequiresRun() + { + var boot = CreateNormalBootstrap(); + Assert.Throws(() => boot.GetConfidenceIntervals(BootstrapCIMethod.Percentile)); + } + + /// + /// Test that the double bootstrap method runs and produces reasonable results. + /// + [TestMethod] + public void Test_DoubleBootstrap() + { + var boot = CreateNormalBootstrap(); + boot.Replicates = 1000; + boot.RunDoubleBootstrap(100); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.Percentile); + + // CIs should bracket the population quantiles + var dist = new Normal(_mu, _sigma); + for (int i = 0; i < _probabilities.Length; i++) + { + double q = dist.InverseCDF(_probabilities[i]); + Assert.IsLessThan(q, results.StatisticResults[i].LowerCI, + $"Lower CI {results.StatisticResults[i].LowerCI} not below quantile {q} at p={_probabilities[i]}"); + Assert.IsGreaterThan(q, results.StatisticResults[i].UpperCI, + $"Upper CI {results.StatisticResults[i].UpperCI} not above quantile {q} at p={_probabilities[i]}"); + } + } + + /// + /// Test that standard error and mean are computed correctly for percentile method. + /// + [TestMethod] + public void Test_StandardErrorAndMean() + { + var boot = CreateNormalBootstrap(); + boot.Replicates = 5000; + boot.Run(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.Percentile); + + // For the median (p=0.5), the SE should be approximately sigma/sqrt(n) ~ 0.0557 + int medianIdx = Array.IndexOf(_probabilities, 0.5); + Assert.IsGreaterThan(0.0, results.StatisticResults[medianIdx].StandardError); + Assert.IsLessThan(0.15, results.StatisticResults[medianIdx].StandardError, + $"SE for median = {results.StatisticResults[medianIdx].StandardError} seems too large"); + + // Mean should be close to the population median + Assert.AreEqual(_mu, results.StatisticResults[medianIdx].Mean, 0.05, + $"Mean for median = {results.StatisticResults[medianIdx].Mean} too far from {_mu}"); + } + + /// + /// Test that FailedReplicates count is zero when no failures occur. + /// + [TestMethod] + public void Test_NoFailures() + { + var boot = CreateNormalBootstrap(); + boot.Replicates = 500; + boot.Run(); + var results = boot.GetConfidenceIntervals(BootstrapCIMethod.Percentile); + + Assert.AreEqual(0, results.FailedReplicates); + for (int i = 0; i < _probabilities.Length; i++) + { + Assert.AreEqual(500, results.StatisticResults[i].ValidCount); + Assert.AreEqual(500, results.StatisticResults[i].TotalCount); + } + } + + } +} diff --git a/Test_Numerics/Sampling/Test_Stratification.cs b/Test_Numerics/Sampling/Test_Stratification.cs index 7ce29f83..9fafac4d 100644 --- a/Test_Numerics/Sampling/Test_Stratification.cs +++ b/Test_Numerics/Sampling/Test_Stratification.cs @@ -117,7 +117,7 @@ public void Test_XToProbability() weights += probs[i].Weight; } // Check weights sum to 1.0 - Assert.AreEqual(1.0d,weights, 1E-8); + Assert.AreEqual(1.0d, weights, 1E-8); } /// @@ -143,7 +143,7 @@ public void Test_XToExceedanceProbability() weights += probs[i].Weight; } // Check weights sum to 1.0 - Assert.AreEqual(1.0d, weights, 1E-8); + Assert.AreEqual(1.0d, weights, 1E-8); } diff --git a/Test_Numerics/Serialization/JsonConverterDemo.cs b/Test_Numerics/Serialization/JsonConverterDemo.cs index c98258fd..b8242acb 100644 --- a/Test_Numerics/Serialization/JsonConverterDemo.cs +++ b/Test_Numerics/Serialization/JsonConverterDemo.cs @@ -33,7 +33,7 @@ using Numerics.Distributions; using Numerics.Utilities; -namespace Test_Numerics.Serialization +namespace Serialization { /// /// Demonstration of custom JSON converters for complex types. diff --git a/Test_Numerics/Test_Numerics.csproj b/Test_Numerics/Test_Numerics.csproj index d25b4db4..abdc9f59 100644 --- a/Test_Numerics/Test_Numerics.csproj +++ b/Test_Numerics/Test_Numerics.csproj @@ -1,7 +1,7 @@  - net481;net8.0;net9.0 + net481;net8.0;net9.0;net10.0 enable false true @@ -20,8 +20,7 @@ - - + diff --git a/Test_Numerics/Utilities/Test_Tools.cs b/Test_Numerics/Utilities/Test_Tools.cs index eed3837c..e0ee963d 100644 --- a/Test_Numerics/Utilities/Test_Tools.cs +++ b/Test_Numerics/Utilities/Test_Tools.cs @@ -183,7 +183,7 @@ public void Test_Standardize() var true_result = new double[] { -0.8164, -0.8164, 0, 0, 1.63299 }; for(int i = 0;i < values.Count; i++) { - Assert.AreEqual(result[i], true_result[i], 1E-04); + Assert.AreEqual(true_result[i], result[i], 1E-04); } } @@ -199,7 +199,7 @@ public void Test_Destandardize() var true_result = new double[] { 3, 3, 4, 4, 6 }; for (int i = 0; i < values.Count; i++) { - Assert.AreEqual(result[i], true_result[i], 1E-03); + Assert.AreEqual(true_result[i], result[i], 1E-03); } } @@ -396,7 +396,7 @@ public void Test_IntegerSequence() var true_result = new int[] { 0, 1, 2, 3 }; for (int i = 0; i < true_result.Length; i++) { - Assert.AreEqual(result[i], true_result[i]); + Assert.AreEqual(true_result[i], result[i]); } } diff --git a/codemeta.json b/codemeta.json new file mode 100644 index 00000000..a9abebe6 --- /dev/null +++ b/codemeta.json @@ -0,0 +1,98 @@ +{ + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "@type": "SoftwareSourceCode", + "name": "Numerics", + "description": "A free and open-source .NET library providing numerical methods, probability distributions, statistical analysis, and Bayesian inference tools for quantitative risk assessment in water resources engineering.", + "version": "2.0.0", + "dateCreated": "2023-09-28", + "dateModified": "2026-03-08", + "license": "https://spdx.org/licenses/BSD-3-Clause", + "codeRepository": "https://github.com/USACE-RMC/Numerics", + "issueTracker": "https://github.com/USACE-RMC/Numerics/issues", + "programmingLanguage": { + "@type": "ComputerLanguage", + "name": "C#", + "url": "https://learn.microsoft.com/en-us/dotnet/csharp/" + }, + "runtimePlatform": ".NET 8.0, 9.0, 10.0, .NET Framework 4.8.1", + "operatingSystem": "Windows, Linux, macOS", + "keywords": [ + "statistics", + "probability distributions", + "MCMC", + "hydrology", + "flood frequency analysis", + "copulas", + "L-moments", + "optimization", + ".NET", + "C#", + "numerical methods", + "Bayesian inference", + "machine learning", + "risk assessment" + ], + "author": [ + { + "@type": "Person", + "givenName": "C. Haden", + "familyName": "Smith", + "@id": "https://orcid.org/0000-0002-4651-9890", + "affiliation": { + "@type": "Organization", + "name": "U.S. Army Corps of Engineers, Risk Management Center" + } + }, + { + "@type": "Person", + "givenName": "Woodrow L.", + "familyName": "Fields", + "affiliation": { + "@type": "Organization", + "name": "U.S. Army Corps of Engineers, Risk Management Center" + } + }, + { + "@type": "Person", + "givenName": "Julian", + "familyName": "Gonzalez", + "affiliation": { + "@type": "Organization", + "name": "U.S. Army Corps of Engineers, Risk Management Center" + } + }, + { + "@type": "Person", + "givenName": "Sadie", + "familyName": "Niblett", + "affiliation": { + "@type": "Organization", + "name": "U.S. Army Corps of Engineers, Risk Management Center" + } + }, + { + "@type": "Person", + "givenName": "Brennan", + "familyName": "Beam", + "affiliation": { + "@type": "Organization", + "name": "U.S. Army Corps of Engineers, Hydrologic Engineering Center" + } + }, + { + "@type": "Person", + "givenName": "Brian", + "familyName": "Skahill", + "@id": "https://orcid.org/0000-0002-2164-0301", + "affiliation": { + "@type": "Organization", + "name": "Fariborz Maseeh Department of Mathematics and Statistics, Portland State University" + } + } + ], + "funder": { + "@type": "Organization", + "name": "U.S. Army Corps of Engineers" + }, + "developmentStatus": "active" +} diff --git a/docs/data/interpolation.md b/docs/data/interpolation.md index e052f486..4f4ff2ac 100644 --- a/docs/data/interpolation.md +++ b/docs/data/interpolation.md @@ -1,24 +1,40 @@ # Data and Interpolation -[← Back to Index](../index.md) +[← Previous: ODE Solvers](../mathematics/ode-solvers.md) | [Back to Index](../index.md) | [Next: Linear Regression →](regression.md) + +Interpolation is the process of estimating values between known data points. Given a set of $n$ data points $(x_1, y_1), (x_2, y_2), \ldots, (x_n, y_n)$, interpolation constructs a function $p(x)$ that passes through all the data points, i.e., $p(x_i) = y_i$ for all $i$. This is distinct from regression, which fits a function that approximates the data while minimizing some error criterion. The ***Numerics*** library provides interpolation methods for estimating values between known data points, essential for data analysis, curve fitting, and function approximation. ## Available Interpolation Methods -| Method | Class | Use Case | -|--------|-------|----------| -| **Linear** | `Linear` | Fast, simple, C⁰ continuous | -| **Cubic Spline** | `CubicSpline` | Smooth curves, C² continuous | -| **Polynomial** | `Polynomial` | Arbitrary order fitting | -| **Bilinear** | `Bilinear` | 2D interpolation on grids | +| Method | Class | Continuity | Error Order | Use Case | +|--------|-------|------------|-------------|----------| +| **Linear** | `Linear` | $C^0$ | $O(h^2)$ | Fast, simple, no overshooting | +| **Cubic Spline** | `CubicSpline` | $C^2$ | $O(h^4)$ | Smooth curves, physical phenomena | +| **Polynomial** | `Polynomial` | $C^\infty$ | Varies | Arbitrary order fitting | +| **Bilinear** | `Bilinear` | $C^0$ | $O(h^2)$ | 2D interpolation on grids | ## Linear Interpolation -Simplest method - connects points with straight lines: +Linear interpolation connects adjacent data points with straight line segments. For a query point $x$ in the interval $[x_i, x_{i+1}]$, the interpolated value is: + +```math +p(x) = y_i + \frac{x - x_i}{x_{i+1} - x_i} \cdot (y_{i+1} - y_i) +``` + +This can be rewritten in terms of the normalized coordinate $t = \frac{x - x_i}{x_{i+1} - x_i}$ as $p(x) = (1-t) \cdot y_i + t \cdot y_{i+1}$, which is simply a weighted average of the two bracketing values. + +**Error bound**: For a function $f(x)$ with bounded second derivative, the interpolation error satisfies: + +```math +|f(x) - p(x)| \leq \frac{h^2}{8} \max |f''(\xi)| +``` + +where $h = x_{i+1} - x_i$ is the local spacing. The error is thus $O(h^2)$, meaning halving the data spacing reduces the error by a factor of four. ```cs -using Numerics.Data.Interpolation; +using Numerics.Data; double[] xData = { 0, 1, 2, 3, 4, 5 }; double[] yData = { 1, 3, 2, 5, 4, 6 }; @@ -41,17 +57,43 @@ for (int i = 0; i < xNew.Length; i++) ``` **Properties:** -- Fast: O(log n) per evaluation -- C⁰ continuous (values continuous, derivatives not) -- No overshooting -- Good for piecewise linear trends +- Fast: $O(\log n)$ per evaluation (bisection search for interval) +- $C^0$ continuous (values continuous, derivatives not) +- No overshooting — interpolant stays within bracket values +- Good for piecewise linear trends or noisy data + +The `Linear` class also supports coordinate transforms via `XTransform` and `YTransform` properties, enabling log-linear or probability-scale interpolation. Available transforms are `Transform.None` (default), `Transform.Logarithmic` ($\log_{10}$), and `Transform.NormalZ` (standard normal quantile). ## Cubic Spline Interpolation -Smooth curves passing through all data points: +A cubic spline constructs a piecewise cubic polynomial $S(x)$ that passes through all data points and has continuous first and second derivatives everywhere. This smoothness is what distinguishes splines from simple piecewise polynomial interpolation. + +### Mathematical Foundation + +On each subinterval $[x_i, x_{i+1}]$, the spline is a cubic polynomial. If we denote the second derivatives at the knot points as $M_i = S''(x_i)$, then the spline on $[x_i, x_{i+1}]$ can be written as: + +```math +S(x) = \frac{M_i}{6h_i}(x_{i+1} - x)^3 + \frac{M_{i+1}}{6h_i}(x - x_i)^3 + \left(\frac{y_i}{h_i} - \frac{M_i h_i}{6}\right)(x_{i+1} - x) + \left(\frac{y_{i+1}}{h_i} - \frac{M_{i+1} h_i}{6}\right)(x - x_i) +``` + +where $h_i = x_{i+1} - x_i$. Requiring continuity of the first derivative $S'(x)$ at each interior knot point $x_i$ (for $i = 1, \ldots, n-2$) yields the tridiagonal system: + +```math +h_{i-1} M_{i-1} + 2(h_{i-1} + h_i) M_i + h_i M_{i+1} = 6 \left( \frac{y_{i+1} - y_i}{h_i} - \frac{y_i - y_{i-1}}{h_{i-1}} \right) +``` + +This system of $n-2$ equations in $n$ unknowns requires two boundary conditions. The ***Numerics*** library uses **natural boundary conditions**, setting $M_0 = 0$ and $M_{n-1} = 0$ (zero second derivatives at the endpoints). The resulting tridiagonal system is solved efficiently using the Thomas algorithm (a specialized form of Gaussian elimination for tridiagonal matrices) in $O(n)$ time [[1]](#1). + +**Error bound**: For a function $f(x)$ with bounded fourth derivative, the natural cubic spline error satisfies: + +```math +|f(x) - S(x)| \leq \frac{5h^4}{384} \max |f^{(4)}(\xi)| +``` + +The $O(h^4)$ convergence rate means halving the data spacing reduces the error by a factor of sixteen — a significant improvement over linear interpolation. ```cs -using Numerics.Data.Interpolation; +using Numerics.Data; double[] xData = { 0, 1, 2, 3, 4, 5 }; double[] yData = { 1, 3, 2, 5, 4, 6 }; @@ -59,79 +101,90 @@ double[] yData = { 1, 3, 2, 5, 4, 6 }; // Natural cubic spline (zero second derivatives at endpoints) var spline = new CubicSpline(xData, yData); -// Interpolate +// Interpolate at a single point double y = spline.Interpolate(2.5); Console.WriteLine($"Spline y(2.5) = {y:F2}"); -// Evaluate derivative -double dy = spline.Differentiate(2.5); -Console.WriteLine($"dy/dx(2.5) = {dy:F2}"); - -// Second derivative -double d2y = spline.Differentiate2(2.5); -Console.WriteLine($"d²y/dx²(2.5) = {d2y:F2}"); +// Interpolate at multiple points +double[] xNew = { 0.5, 1.5, 2.5, 3.5 }; +double[] yNew = spline.Interpolate(xNew); +for (int i = 0; i < xNew.Length; i++) +{ + Console.WriteLine($" y({xNew[i]}) = {yNew[i]:F2}"); +} ``` **Properties:** -- C² continuous (smooth second derivative) +- $C^2$ continuous (smooth second derivative) - Unique solution through all points -- Natural boundary conditions -- May overshoot between points +- Natural boundary conditions ($S''=0$ at endpoints) +- May overshoot between data points, especially with oscillatory data - Excellent for smooth physical phenomena -### Boundary Conditions +## Polynomial Interpolation + +Given $n$ data points, there exists a unique polynomial of degree at most $n-1$ that passes through all of them. However, fitting a single high-degree polynomial to many data points is often a poor choice due to Runge's phenomenon (discussed below). -```cs -// Natural spline (second derivative = 0 at endpoints) -var naturalSpline = new CubicSpline(xData, yData, - boundaryType: CubicSpline.BoundaryType.Natural); - -// Clamped spline (specify first derivatives at endpoints) -double leftDerivative = 0.5; -double rightDerivative = 0.8; -var clampedSpline = new CubicSpline(xData, yData, - boundaryType: CubicSpline.BoundaryType.Clamped, - leftBoundaryValue: leftDerivative, - rightBoundaryValue: rightDerivative); - -// Not-a-knot spline (third derivative continuous at second and penultimate points) -var notAKnotSpline = new CubicSpline(xData, yData, - boundaryType: CubicSpline.BoundaryType.NotAKnot); +### Neville's Method + +The ***Numerics*** library uses Neville's method [[1]](#1) to evaluate the interpolating polynomial. Rather than computing coefficients explicitly, Neville's method builds a tableau of progressively higher-degree polynomial approximations through recursive divided differences: + +```math +P_{i,j}(x) = \frac{(x - x_j) P_{i,j-1}(x) - (x - x_i) P_{i+1,j}(x)}{x_i - x_j} ``` -## Polynomial Interpolation +where $P_{i,i}(x) = y_i$ are the initial zeroth-degree polynomials. The method also provides an error estimate from the last correction term added to the approximation. -Fits polynomial of specified degree: +The `Polynomial` class fits a polynomial of specified order using a local window of `order + 1` points centered near the query point, which helps mitigate oscillation issues: ```cs -using Numerics.Data.Interpolation; +using Numerics.Data; double[] xData = { 0, 1, 2, 3, 4 }; double[] yData = { 1, 3, 2, 5, 4 }; -// Fit 3rd degree polynomial -var poly = new Polynomial(xData, yData, degree: 3); +// Fit 3rd order polynomial (order is the first parameter) +var poly = new Polynomial(3, xData, yData); // Interpolate double y = poly.Interpolate(2.5); Console.WriteLine($"Polynomial y(2.5) = {y:F2}"); -// Get polynomial coefficients -double[] coeffs = poly.Coefficients; -Console.WriteLine("Polynomial: y = " + - string.Join(" + ", coeffs.Select((c, i) => $"{c:F3}x^{i}"))); +// The error estimate from the most recent interpolation +Console.WriteLine($"Error estimate: {poly.Error:F6}"); ``` -**Warning:** High-degree polynomials (> 5) can exhibit Runge's phenomenon (oscillations). +### Runge's Phenomenon + +A critical limitation of polynomial interpolation is Runge's phenomenon: as the polynomial degree increases, the interpolation error can grow dramatically near the edges of the interval, even for smooth functions. Consider the Runge function: + +```math +f(x) = \frac{1}{1 + 25x^2} +``` + +With $n$ equally spaced points on $[-1, 1]$, the degree $n-1$ interpolating polynomial oscillates with increasing amplitude near $x = \pm 1$ as $n$ grows. For $n = 11$ (degree 10), the maximum error near the boundaries exceeds 1.0, even though $f(x)$ is perfectly smooth. + +**Mitigation strategies:** +- Use **cubic splines** instead of high-degree polynomials — splines keep each piece low-degree while maintaining global smoothness +- Use **Chebyshev nodes** (non-uniform spacing clustered near endpoints) if you can control where data is collected +- Keep polynomial degree low ($\leq 5$) and use a local window, as the `Polynomial` class does **Best practice:** Use splines instead of high-degree polynomials. ## Bilinear Interpolation -For 2D gridded data: +Bilinear interpolation extends linear interpolation to two-dimensional gridded data. For a query point $(x, y)$ in the cell bounded by grid points $(x_i, y_j)$, $(x_{i+1}, y_j)$, $(x_i, y_{j+1})$, $(x_{i+1}, y_{j+1})$, the interpolated value is computed by performing linear interpolation twice — first along one axis, then along the other: + +```math +z(x, y) = (1-t)(1-u) \cdot z_{i,j} + t(1-u) \cdot z_{i+1,j} + t \cdot u \cdot z_{i+1,j+1} + (1-t) \cdot u \cdot z_{i,j+1} +``` + +where $t = \frac{x - x_i}{x_{i+1} - x_i}$ and $u = \frac{y - y_j}{y_{j+1} - y_j}$ are the normalized coordinates within the cell. The result is independent of the order in which the two linear interpolations are performed. + +**Error**: For a function with bounded mixed partial derivative, $|f(x,y) - z(x,y)| = O(h^2)$, similar to 1D linear interpolation. ```cs -using Numerics.Data.Interpolation; +using Numerics.Data; // Grid coordinates double[] xGrid = { 0, 1, 2 }; @@ -150,17 +203,19 @@ var bilinear = new Bilinear(xGrid, yGrid, zGrid); double z = bilinear.Interpolate(0.5, 0.5); Console.WriteLine($"z(0.5, 0.5) = {z:F2}"); -// Multiple points +// Multiple points (loop over individual point pairs) double[] xNew = { 0.5, 1.5 }; double[] yNew = { 0.5, 1.5 }; -double[] zNew = bilinear.Interpolate(xNew, yNew); for (int i = 0; i < xNew.Length; i++) { - Console.WriteLine($"z({xNew[i]}, {yNew[i]}) = {zNew[i]:F2}"); + double zi = bilinear.Interpolate(xNew[i], yNew[i]); + Console.WriteLine($"z({xNew[i]}, {yNew[i]}) = {zi:F2}"); } ``` +Like the `Linear` class, `Bilinear` supports coordinate transforms (`X1Transform`, `X2Transform`, `YTransform`) for log-linear or probability-scale interpolation in 2D. + **Applications:** - Image resizing - Terrain elevation maps @@ -228,9 +283,11 @@ Console.WriteLine($"At x = {testPoint}:"); Console.WriteLine($" Linear: {yLinear:F3}"); Console.WriteLine($" Spline: {ySpline:F3}"); Console.WriteLine("\nLinear connects with straight line"); -Console.WriteLine("Spline creates smooth curve"); +Console.WriteLine("Spline creates smooth curve (may overshoot)"); ``` +Note the spline may produce values outside the range of the bracketing data points due to the smoothness constraint. For data that must remain monotonic, this overshoot can be problematic — in such cases, consider using linear interpolation instead. + ### Example 4: 2D Surface Interpolation ```cs @@ -251,41 +308,56 @@ double z = terrain.Interpolate(x, y); Console.WriteLine($"Terrain elevation at ({x}, {y}): {z:F1} m"); -// Create contour at specific elevation -double contourElevation = 110; -Console.WriteLine($"\nFinding points at {contourElevation} m elevation..."); -// Would need to scan grid and find where z = contourElevation +// Sample elevations along a transect +Console.WriteLine("\nElevation transect from (2,2) to (8,8):"); +for (double t = 0; t <= 1; t += 0.2) +{ + double xi = 2 + 6 * t; + double yi = 2 + 6 * t; + double zi = terrain.Interpolate(xi, yi); + Console.WriteLine($" ({xi:F1}, {yi:F1}): {zi:F1} m"); +} ``` ## Best Practices -1. **Data spacing**: Interpolation works best with reasonably uniform spacing -2. **Extrapolation**: Avoid extrapolating beyond data range (highly unreliable) -3. **Smoothness**: Use splines for smooth physical phenomena, linear for piecewise trends -4. **Outliers**: Check for data errors before interpolating -5. **Monotonicity**: If data should be monotonic, consider specialized methods -6. **Periodic data**: Consider Fourier or trigonometric interpolation +1. **Data spacing**: Interpolation works best with reasonably uniform spacing. Highly irregular spacing can lead to large errors in some subintervals +2. **Extrapolation**: Avoid extrapolating beyond data range — splines and polynomials are especially unreliable outside the data bounds. The `Linear` class returns boundary values; the `Bilinear` class falls back to 1D interpolation at edges +3. **Smoothness**: Use splines for smooth physical phenomena ($C^2$ continuity), linear for piecewise trends or noisy data ($C^0$ continuity with no overshooting) +4. **Outliers**: Check for data errors before interpolating — splines will faithfully pass through outliers +5. **Monotonicity**: Cubic splines do not preserve monotonicity of the data. If your data should be monotonic (e.g., a CDF or rating curve), verify the interpolant doesn't violate this +6. **Periodic data**: Consider Fourier or trigonometric interpolation for periodic signals ## Choosing an Interpolation Method -| Data Characteristics | Recommended Method | -|---------------------|-------------------| -| Few points, simple trend | Linear | -| Smooth physical process | Cubic Spline | -| Need derivatives | Cubic Spline | -| Noisy data | Linear or smoothing spline | -| 2D regular grid | Bilinear | -| Piecewise constant | Nearest neighbor | -| Exact polynomial | Polynomial (low degree) | +| Data Characteristics | Recommended Method | Why | +|---------------------|-------------------|-----| +| Few points, simple trend | Linear | No overshooting, $O(h^2)$ error | +| Smooth physical process | Cubic Spline | $C^2$ smooth, $O(h^4)$ error | +| Need derivatives | Cubic Spline | Spline derivatives are well-defined | +| Noisy data | Linear | Splines amplify noise through curvature matching | +| 2D regular grid | Bilinear | Direct extension to 2D | +| Piecewise constant | Nearest neighbor | Preserves step structure | +| Exact polynomial | Polynomial (low degree) | Neville's method with error estimate | ## Common Pitfalls -1. **Runge's phenomenon**: High-degree polynomials oscillate wildly -2. **Extrapolation**: Results outside data range are unreliable -3. **Unequal spacing**: Some methods assume uniform spacing -4. **Monotonicity**: Splines may violate monotonicity of data -5. **Edge effects**: Interpolation near boundaries less accurate +1. **Runge's phenomenon**: High-degree polynomials ($>5$) oscillate wildly near interval boundaries — use splines instead +2. **Extrapolation**: Results outside the data range are unreliable for all methods +3. **Natural boundary conditions**: The zero-curvature constraint at endpoints can cause the spline to flatten near the boundaries, which may be physically inappropriate +4. **Monotonicity violation**: Cubic splines can introduce local extrema between data points, violating monotonicity +5. **Edge effects**: All methods lose accuracy near the boundaries of the data range + +--- + +## References + +[1] W. H. Press, S. A. Teukolsky, W. T. Vetterling and B. P. Flannery, *Numerical Recipes: The Art of Scientific Computing*, 3rd ed., Cambridge, UK: Cambridge University Press, 2007. + +[2] C. de Boor, *A Practical Guide to Splines*, Rev. ed., New York: Springer, 2001. + +[3] R. L. Burden and J. D. Faires, *Numerical Analysis*, 9th ed., Boston: Brooks/Cole, 2010. --- -[← Back to Index](../index.md) +[← Previous: ODE Solvers](../mathematics/ode-solvers.md) | [Back to Index](../index.md) | [Next: Linear Regression →](regression.md) diff --git a/docs/data/regression.md b/docs/data/regression.md new file mode 100644 index 00000000..ad8c7a5e --- /dev/null +++ b/docs/data/regression.md @@ -0,0 +1,376 @@ +# Linear Regression + +[← Previous: Interpolation](interpolation.md) | [Back to Index](../index.md) | [Next: Time Series →](time-series.md) + +Linear regression estimates the relationship between a scalar response variable $y$ and one or more predictor variables $\mathbf{x}$. The model assumes a linear form: + +```math +y_i = \beta_0 + \beta_1 x_{i1} + \beta_2 x_{i2} + \cdots + \beta_p x_{ip} + \varepsilon_i +``` + +where $\beta_0$ is the intercept, $\beta_1, \ldots, \beta_p$ are the regression coefficients, and $\varepsilon_i \sim N(0, \sigma^2)$ are independent error terms. In matrix notation, this becomes $\mathbf{y} = \mathbf{X}\boldsymbol{\beta} + \boldsymbol{\varepsilon}$. + +## The Normal Equations + +The ordinary least squares (OLS) estimator minimizes the sum of squared residuals $\sum_{i=1}^n (y_i - \hat{y}_i)^2$. Setting the gradient of this objective to zero yields the **normal equations**: + +```math +\mathbf{X}^T \mathbf{X} \boldsymbol{\hat{\beta}} = \mathbf{X}^T \mathbf{y} +``` + +When $\mathbf{X}^T \mathbf{X}$ is invertible, the solution is: + +```math +\boldsymbol{\hat{\beta}} = (\mathbf{X}^T \mathbf{X})^{-1} \mathbf{X}^T \mathbf{y} +``` + +However, directly computing $(\mathbf{X}^T \mathbf{X})^{-1}$ is numerically ill-conditioned, especially when predictor variables are correlated or span very different scales. The condition number of $\mathbf{X}^T \mathbf{X}$ is the square of the condition number of $\mathbf{X}$, so even moderate collinearity can cause significant loss of precision. + +### Why SVD? + +The ***Numerics*** library uses **Singular Value Decomposition** (SVD) to solve the least squares problem instead [[1]](#1). The SVD factorizes the design matrix as $\mathbf{X} = \mathbf{U} \mathbf{W} \mathbf{V}^T$, where $\mathbf{U}$ and $\mathbf{V}$ are orthogonal matrices and $\mathbf{W}$ is a diagonal matrix of singular values. The least squares solution is then: + +```math +\boldsymbol{\hat{\beta}} = \mathbf{V} \mathbf{W}^{-1} \mathbf{U}^T \mathbf{y} +``` + +SVD provides several advantages: +- **Numerical stability**: Works correctly even when $\mathbf{X}^T \mathbf{X}$ is nearly singular +- **Rank detection**: Small singular values (below a threshold of $10^{-12}$) are set to zero, effectively handling rank-deficient problems +- **Covariance computation**: The parameter covariance matrix is computed directly from the SVD components as $\text{Cov}(\hat{\beta}_{i}, \hat{\beta}_{j}) = \sigma^2 \sum_k \frac{V_{ik} V_{jk}}{W_k^2}$ + +## Creating a Linear Regression Model + +The model is fitted automatically in the constructor — no separate `Train()` call is needed: + +```cs +using Numerics.Data; +using Numerics.Mathematics.LinearAlgebra; + +// Predictor data (no intercept column needed — added automatically) +double[,] X = { + { 1.0 }, + { 2.0 }, + { 3.0 }, + { 4.0 }, + { 5.0 } +}; + +double[] y = { 2.1, 4.0, 5.8, 8.1, 9.9 }; + +// Create and fit the model (fitting happens in the constructor) +var lm = new LinearRegression(new Matrix(X), new Vector(y)); + +Console.WriteLine($"Intercept: {lm.Parameters[0]:F4}"); +Console.WriteLine($"Slope: {lm.Parameters[1]:F4}"); +Console.WriteLine($"R²: {lm.RSquared:F4}"); +Console.WriteLine($"Adjusted R²: {lm.AdjRSquared:F4}"); +Console.WriteLine($"Standard Error: {lm.StandardError:F4}"); +``` + +## Model Properties + +After construction, the following properties are available: + +| Property | Type | Description | +|----------|------|-------------| +| `Parameters` | `List` | Estimated coefficients (intercept first if `HasIntercept = true`) | +| `ParameterNames` | `List` | Names of each parameter | +| `ParameterStandardErrors` | `List` | Standard errors of each coefficient | +| `ParameterTStats` | `List` | t-statistics for each coefficient | +| `Covariance` | `Matrix` | Parameter covariance matrix | +| `Residuals` | `double[]` | Model residuals (observed − predicted) | +| `StandardError` | `double` | Residual standard error | +| `SampleSize` | `int` | Number of observations | +| `DegreesOfFreedom` | `int` | Residual degrees of freedom ($n - p$) | +| `RSquared` | `double` | Coefficient of determination | +| `AdjRSquared` | `double` | Adjusted $R^2$ | +| `HasIntercept` | `bool` | Whether the model includes an intercept | + +### Understanding Key Statistics + +**Coefficient of determination** ($R^2$) measures the proportion of variance in $y$ explained by the model: + +```math +R^2 = 1 - \frac{\sum (y_i - \hat{y}_i)^2}{\sum (y_i - \bar{y})^2} = 1 - \frac{SS_{res}}{SS_{tot}} +``` + +**Adjusted $R^2$** penalizes for additional predictors, preventing overfitting: + +```math +R^2_{adj} = 1 - \frac{SS_{res} / (n - p)}{SS_{tot} / (n - 1)} +``` + +where $n$ is the sample size and $p$ is the number of parameters (including intercept). + +**Parameter standard errors** quantify uncertainty in each coefficient estimate. The standard error of $\hat{\beta}_j$ is: + +```math +SE(\hat{\beta}_j) = \hat{\sigma} \sqrt{C_{jj}} +``` + +where $\hat{\sigma}$ is the residual standard error and $C_{jj}$ is the $j$-th diagonal element of $(\mathbf{X}^T \mathbf{X})^{-1}$ (computed via SVD in practice). + +**t-statistics** test whether each coefficient is significantly different from zero: $t_j = \hat{\beta}_j / SE(\hat{\beta}_j)$. Under the null hypothesis $\beta_j = 0$, this follows a Student's t-distribution with $n - p$ degrees of freedom. + +## Summary Output + +The `Summary()` method produces an R-style summary table: + +```cs +using Numerics.Data; +using Numerics.Mathematics.LinearAlgebra; + +double[,] X = { + { 12.5 }, { 25.0 }, { 48.3 }, { 75.0 }, { 102.0 }, + { 18.7 }, { 55.0 }, { 130.0 }, { 8.2 }, { 200.0 } +}; + +double[] y = { 1450, 2680, 4520, 6100, 7850, 2050, 5200, 9300, 980, 12500 }; + +var lm = new LinearRegression(new Matrix(X), new Vector(y)); + +// Print R-style summary +foreach (var line in lm.Summary()) +{ + Console.WriteLine(line); +} +``` + +The summary includes: +- Coefficient estimates with standard errors, t-statistics, and p-values +- Significance codes (*** < 0.001, ** < 0.01, * < 0.05) +- Residual standard error and degrees of freedom +- $R^2$ and adjusted $R^2$ +- F-statistic with p-value +- Five-number summary of residuals + +## Prediction + +### Point Predictions + +```cs +// Predict for new predictor values +double[,] XNew = { + { 50.0 }, + { 100.0 }, + { 150.0 } +}; + +double[] predictions = lm.Predict(new Matrix(XNew)); + +Console.WriteLine("Predictions:"); +for (int i = 0; i < predictions.Length; i++) +{ + Console.WriteLine($" X = {XNew[i, 0]:F1} → ŷ = {predictions[i]:F1}"); +} +``` + +### Prediction Intervals + +Prediction intervals account for both the uncertainty in the estimated regression line and the inherent variability of individual observations: + +```math +\hat{y}_0 \pm t_{\alpha/2, n-p} \cdot \hat{\sigma} \sqrt{1 + \mathbf{x}_0^T (\mathbf{X}^T \mathbf{X})^{-1} \mathbf{x}_0} +``` + +The $1$ inside the square root represents the irreducible variance of a new observation; the second term represents uncertainty in the estimated mean. + +```cs +// 90% prediction intervals (alpha = 0.1) +double[,] intervals = lm.PredictionIntervals(new Matrix(XNew), alpha: 0.1); + +Console.WriteLine("90% Prediction Intervals:"); +Console.WriteLine(" X | Lower | Mean | Upper"); +Console.WriteLine("-----------|-----------|-----------|----------"); + +for (int i = 0; i < XNew.GetLength(0); i++) +{ + Console.WriteLine($" {XNew[i, 0],7:F1} | {intervals[i, 0],9:F1} | {intervals[i, 2],9:F1} | {intervals[i, 1],9:F1}"); +} +``` + +**Note:** `PredictionIntervals` returns a 2D array with columns: lower (index 0), upper (index 1), mean (index 2). + +## Multiple Regression + +Multiple predictor variables are supported: + +```cs +using Numerics.Data; +using Numerics.Mathematics.LinearAlgebra; + +// Three predictor variables +double[,] X = { + { 12.5, 38.2, 820 }, + { 25.0, 40.1, 790 }, + { 48.3, 39.5, 850 }, + { 75.0, 41.0, 780 }, + { 102.0, 42.3, 760 }, + { 130.0, 43.0, 740 }, + { 200.0, 44.2, 720 } +}; + +double[] y = { 1450, 2680, 4520, 6100, 7850, 9300, 12500 }; + +var lm = new LinearRegression(new Matrix(X), new Vector(y)); + +Console.WriteLine("Multiple Regression Results:"); +for (int i = 0; i < lm.Parameters.Count; i++) +{ + Console.WriteLine($" {lm.ParameterNames[i]}: {lm.Parameters[i]:F4} (SE: {lm.ParameterStandardErrors[i]:F4})"); +} + +Console.WriteLine($"\nR²: {lm.RSquared:F4}"); +Console.WriteLine($"Adjusted R²: {lm.AdjRSquared:F4}"); +``` + +### Multicollinearity + +When predictor variables are highly correlated, the regression coefficients become unstable — small changes in the data lead to large changes in the estimated coefficients, and standard errors inflate. This is called **multicollinearity**. + +Signs of multicollinearity: +- Large standard errors on coefficients that should be significant +- $R^2$ is high but individual t-statistics are low +- Coefficient signs are opposite to what domain knowledge suggests + +One diagnostic is the **Variance Inflation Factor** (VIF), defined for each predictor $j$ as: + +```math +\text{VIF}_j = \frac{1}{1 - R^2_j} +``` + +where $R^2_j$ is the $R^2$ from regressing $x_j$ on all other predictors. A VIF above 10 suggests problematic collinearity. While the ***Numerics*** library does not compute VIF directly, you can fit auxiliary regressions to diagnose it. + +## Regression Without Intercept + +```cs +// Force model through the origin (no intercept) +var lm = new LinearRegression(new Matrix(X), new Vector(y), hasIntercept: false); + +Console.WriteLine("No-intercept model:"); +Console.WriteLine($"Slope: {lm.Parameters[0]:F4}"); +``` + +**Caution**: Removing the intercept changes the interpretation of $R^2$ and can bias coefficients if the true relationship does not pass through the origin. Only use this when the physics of the problem demands it. + +## Regression Assumptions + +The validity of OLS inference (standard errors, p-values, prediction intervals) depends on several assumptions about the error terms $\varepsilon_i$: + +1. **Linearity**: The relationship between $\mathbf{x}$ and $y$ is linear. Check by plotting residuals vs. fitted values — a systematic pattern indicates nonlinearity. + +2. **Independence**: Errors are independent. Violated with time series data or spatial data. The Ljung-Box test (available in ***Numerics*** via `HypothesisTests.LjungBoxTest`) can detect serial correlation. + +3. **Homoscedasticity**: Errors have constant variance $\text{Var}(\varepsilon_i) = \sigma^2$. A fan-shaped pattern in the residual plot indicates heteroscedasticity. Consider log-transforming the response variable if variance increases with the mean. + +4. **Normality**: Errors are normally distributed. Check with the Jarque-Bera test (available via `HypothesisTests.JarqueBeraTest`). Mild departures from normality have little impact on coefficient estimates but affect confidence intervals and p-values. + +## Practical Examples + +### Example 1: Regional Streamflow Regression + +Predicting annual peak streamflow from watershed characteristics using data from [`example-data/streamflow-regression.csv`](../example-data/streamflow-regression.csv): + +```cs +using System.IO; +using System.Linq; +using Numerics.Data; +using Numerics.Mathematics.LinearAlgebra; + +// Load CSV data (skip comment lines starting with #) +string[] lines = File.ReadAllLines("example-data/streamflow-regression.csv"); +var dataLines = lines + .Where(line => !line.StartsWith("#") && !string.IsNullOrWhiteSpace(line)) + .Skip(1) // Skip header + .ToArray(); + +int n = dataLines.Length; +double[,] features = new double[n, 3]; +double[] flow = new double[n]; + +for (int i = 0; i < n; i++) +{ + var parts = dataLines[i].Split(','); + features[i, 0] = double.Parse(parts[0]); // DrainageArea_sqmi + features[i, 1] = double.Parse(parts[1]); // MeanAnnualPrecip_in + features[i, 2] = double.Parse(parts[2]); // MeanElevation_ft + flow[i] = double.Parse(parts[3]); // AnnualPeakFlow_cfs +} + +// Fit regression model +var lm = new LinearRegression(new Matrix(features), new Vector(flow)); + +// Print summary +Console.WriteLine("Regional Streamflow Regression"); +Console.WriteLine("=" + new string('=', 50)); +foreach (var line in lm.Summary()) +{ + Console.WriteLine(line); +} + +// Predict for a new watershed +double[,] newSite = { { 100.0, 41.0, 780 } }; +double[] predicted = lm.Predict(new Matrix(newSite)); +double[,] interval = lm.PredictionIntervals(new Matrix(newSite), alpha: 0.1); + +Console.WriteLine($"\nPrediction for 100 sq mi watershed:"); +Console.WriteLine($" Predicted peak flow: {predicted[0]:F0} cfs"); +Console.WriteLine($" 90% Prediction interval: [{interval[0, 0]:F0}, {interval[0, 1]:F0}] cfs"); +``` + +### Example 2: Residual Analysis + +```cs +using Numerics.Data; +using Numerics.Data.Statistics; +using Numerics.Mathematics.LinearAlgebra; + +double[,] X = { + { 12.5 }, { 25.0 }, { 48.3 }, { 75.0 }, { 102.0 }, + { 18.7 }, { 55.0 }, { 130.0 }, { 8.2 }, { 200.0 } +}; + +double[] y = { 1450, 2680, 4520, 6100, 7850, 2050, 5200, 9300, 980, 12500 }; + +var lm = new LinearRegression(new Matrix(X), new Vector(y)); + +// Residual diagnostics +Console.WriteLine("Residual Diagnostics:"); +Console.WriteLine($" Mean residual: {lm.Residuals.Average():F2}"); +Console.WriteLine($" Std dev of residuals: {Statistics.StandardDeviation(lm.Residuals):F2}"); +Console.WriteLine($" Min residual: {lm.Residuals.Min():F2}"); +Console.WriteLine($" Max residual: {lm.Residuals.Max():F2}"); + +// Check normality of residuals +double jbPValue = HypothesisTests.JarqueBeraTest(lm.Residuals); +Console.WriteLine($"\nJarque-Bera test p-value: {jbPValue:F4}"); + +if (jbPValue > 0.05) + Console.WriteLine("Residuals are consistent with normality (p > 0.05)"); +else + Console.WriteLine("Residuals may not be normally distributed (p < 0.05)"); +``` + +## Best Practices + +1. **Check assumptions** — Plot residuals vs. fitted values to detect nonlinearity and heteroscedasticity. Test residual normality with Jarque-Bera +2. **Examine residuals** — Residuals should appear random with no systematic pattern. Look for outliers (residuals beyond $\pm 3\sigma$) +3. **Watch for multicollinearity** — Highly correlated predictors inflate standard errors and destabilize coefficients. Compute VIF if suspected +4. **Use adjusted $R^2$** — Better for comparing models with different numbers of predictors, as it penalizes model complexity +5. **Validate predictions** — Don't extrapolate far beyond the range of training data. Prediction intervals widen rapidly outside the data range +6. **Consider transformations** — Log-transform skewed variables before fitting. For nonlinear relationships, consider the GLM framework available in the [Machine Learning](../machine-learning.md) documentation + +--- + +## References + +[1] W. H. Press, S. A. Teukolsky, W. T. Vetterling and B. P. Flannery, *Numerical Recipes: The Art of Scientific Computing*, 3rd ed., Cambridge, UK: Cambridge University Press, 2007. + +[2] G. H. Golub and C. F. Van Loan, *Matrix Computations*, 4th ed., Baltimore: Johns Hopkins University Press, 2013. + +[3] R. L. Burden and J. D. Faires, *Numerical Analysis*, 9th ed., Boston: Brooks/Cole, 2010. + +--- + +[← Previous: Interpolation](interpolation.md) | [Back to Index](../index.md) | [Next: Time Series →](time-series.md) diff --git a/docs/data/time-series.md b/docs/data/time-series.md index 11f1055c..62296722 100644 --- a/docs/data/time-series.md +++ b/docs/data/time-series.md @@ -1,6 +1,6 @@ # Time Series -[← Previous: Interpolation](interpolation.md) | [Back to Index](../index.md) | [Next: Random Generation →](../sampling/random-generation.md) +[← Previous: Linear Regression](regression.md) | [Back to Index](../index.md) | [Next: Descriptive Statistics →](../statistics/descriptive.md) The ***Numerics*** library provides a comprehensive `TimeSeries` class for working with time-indexed data. This class supports regular and irregular time intervals, statistical operations, transformations, and analysis methods essential for hydrological and environmental data. @@ -15,8 +15,8 @@ using Numerics.Data; var ts = new TimeSeries(); // Create with time interval -var dailyData = new TimeSeries(TimeInterval.Daily); -var monthlyData = new TimeSeries(TimeInterval.Monthly); +var dailyData = new TimeSeries(TimeInterval.OneDay); +var monthlyData = new TimeSeries(TimeInterval.OneMonth); ``` ### Time Series with Date Range @@ -26,11 +26,11 @@ var monthlyData = new TimeSeries(TimeInterval.Monthly); DateTime start = new DateTime(2020, 1, 1); DateTime end = new DateTime(2020, 12, 31); -// With NaN values (placeholder) -var ts1 = new TimeSeries(TimeInterval.Daily, start, end); +// Create empty time series (values default to NaN) +var ts1 = new TimeSeries(TimeInterval.OneDay, start, end); // With fixed value -var ts2 = new TimeSeries(TimeInterval.Daily, start, end, fixedValue: 0.0); +var ts2 = new TimeSeries(TimeInterval.OneDay, start, end, fixedValue: 0.0); Console.WriteLine($"Created time series with {ts1.Count} daily values"); ``` @@ -42,7 +42,7 @@ Console.WriteLine($"Created time series with {ts1.Count} daily values"); double[] dailyFlow = { 125.0, 130.0, 135.0, 132.0, 138.0 }; DateTime start = new DateTime(2024, 1, 1); -var ts = new TimeSeries(TimeInterval.Daily, start, dailyFlow); +var ts = new TimeSeries(TimeInterval.OneDay, start, dailyFlow); Console.WriteLine("Daily Flow Data:"); for (int i = 0; i < ts.Count; i++) @@ -58,24 +58,24 @@ Supported time intervals: ```cs public enum TimeInterval { - Irregular, // No fixed interval OneMinute, FiveMinute, - TenMinute, FifteenMinute, ThirtyMinute, OneHour, SixHour, TwelveHour, - Daily, - Weekly, - Monthly, - Annual + OneDay, + SevenDay, + OneMonth, + OneQuarter, + OneYear, + Irregular // No fixed interval } // Example usage var hourlyData = new TimeSeries(TimeInterval.OneHour); -var yearlyData = new TimeSeries(TimeInterval.Annual); +var yearlyData = new TimeSeries(TimeInterval.OneYear); ``` ## Accessing Data @@ -83,23 +83,26 @@ var yearlyData = new TimeSeries(TimeInterval.Annual); ### Indexing ```cs -var ts = new TimeSeries(TimeInterval.Daily, new DateTime(2024, 1, 1), +using System.Linq; +using Numerics.Data; + +var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(2024, 1, 1), new[] { 10.0, 15.0, 20.0, 25.0, 30.0 }); // Access by index double value = ts[2].Value; // 20.0 DateTime date = ts[2].Index; // 2024-01-03 -// Access by date +// Access by date (use LINQ to find ordinate by date) DateTime queryDate = new DateTime(2024, 1, 3); -var ordinate = ts.GetByIndex(queryDate); +var ordinate = ts.First(o => o.Index == queryDate); Console.WriteLine($"Flow on {queryDate:yyyy-MM-dd}: {ordinate.Value:F1}"); // Properties int count = ts.Count; -DateTime firstDate = ts.FirstIndex; -DateTime lastDate = ts.LastIndex; -double[] values = ts.Values.ToArray(); +DateTime firstDate = ts.StartDate; +DateTime lastDate = ts.EndDate; +double[] values = ts.ValuesToArray(); ``` ### Missing Values @@ -126,7 +129,7 @@ for (int i = 0; i < ts.Count; i++) ### Basic Arithmetic ```cs -var ts = new TimeSeries(TimeInterval.Daily, new DateTime(2024, 1, 1), +var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(2024, 1, 1), new[] { 10.0, 15.0, 20.0, 25.0, 30.0 }); // Add constant to all values @@ -161,7 +164,7 @@ ts.Multiply(1.5, indexes); // Multiply selected values by 1.5 ### Transformations ```cs -var ts = new TimeSeries(TimeInterval.Daily, new DateTime(2024, 1, 1), +var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(2024, 1, 1), new[] { 10.0, 15.0, 20.0, 25.0, 30.0 }); // Absolute value @@ -182,11 +185,31 @@ ts.Inverse(); // 1 / x ## Time Series Analysis +### Autocorrelation + +The **autocorrelation function** (ACF) measures the correlation of a time series with a lagged copy of itself. At lag $k$, the sample autocorrelation is: + +```math +\hat{\rho}(k) = \frac{\sum_{t=1}^{n-k}(x_t - \bar{x})(x_{t+k} - \bar{x})}{\sum_{t=1}^{n}(x_t - \bar{x})^2} +``` + +where $\bar{x}$ is the sample mean and $n$ is the series length. By definition, $\hat{\rho}(0) = 1$. + +Autocorrelation is central to time series analysis: significant autocorrelation at lag $k$ indicates that values $k$ time steps apart are linearly related. For an independent series, $\hat{\rho}(k) \approx 0$ for all $k > 0$, and the approximate 95% confidence bounds are $\pm 1.96 / \sqrt{n}$. + +The ***Numerics*** library uses the ACF internally in several statistical tests. The **Ljung-Box test** (`SummaryHypothesisTest()`) checks whether a group of autocorrelations are jointly significant: + +```math +Q = n(n+2) \sum_{k=1}^{h} \frac{\hat{\rho}(k)^2}{n - k} +``` + +Under the null hypothesis of independence, $Q \sim \chi^2(h)$. + ### Cumulative Sum ```cs double[] dailyRainfall = { 0.5, 1.2, 0.8, 0.0, 2.1, 1.5 }; -var rainfall = new TimeSeries(TimeInterval.Daily, new DateTime(2024, 1, 1), dailyRainfall); +var rainfall = new TimeSeries(TimeInterval.OneDay, new DateTime(2024, 1, 1), dailyRainfall); // Compute cumulative rainfall var cumulative = rainfall.CumulativeSum(); @@ -200,6 +223,14 @@ for (int i = 0; i < rainfall.Count; i++) ### Differencing +The **difference operator** $\nabla$ removes trends from a time series. The first difference at lag $d$ is: + +```math +\nabla_d x_t = x_t - x_{t-d} +``` + +First differencing ($d = 1$) removes a linear trend. Applying the operator twice ($\nabla^2 x_t = \nabla(\nabla x_t)$) removes a quadratic trend. Seasonal differencing uses a lag equal to the seasonal period — for example, $d = 12$ for monthly data with an annual cycle removes the seasonal component directly. + ```cs // First difference (change from previous) var diff1 = ts.Difference(lag: 1, differences: 1); @@ -226,26 +257,28 @@ for (int i = 0; i < Math.Min(5, ts.Count); i++) ### Descriptive Statistics ```cs +using System.Linq; +using Numerics.Data; using Numerics.Data.Statistics; -var ts = new TimeSeries(TimeInterval.Daily, new DateTime(2024, 1, 1), +var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(2024, 1, 1), new[] { 125.0, 130.0, 135.0, 132.0, 138.0, 145.0 }); // Compute statistics -double mean = Statistics.Mean(ts.Values.ToArray()); -double std = Statistics.StandardDeviation(ts.Values.ToArray()); -double min = ts.Values.Min(); -double max = ts.Values.Max(); +double mean = Statistics.Mean(ts.ValuesToArray()); +double std = Statistics.StandardDeviation(ts.ValuesToArray()); +double min = ts.ValuesToArray().Min(); +double max = ts.ValuesToArray().Max(); Console.WriteLine($"Mean: {mean:F1}"); Console.WriteLine($"Std Dev: {std:F1}"); Console.WriteLine($"Range: [{min:F1}, {max:F1}]"); // Percentiles -double[] values = ts.Values.ToArray(); -double p25 = Statistics.Percentile(values, 25); -double p50 = Statistics.Percentile(values, 50); -double p75 = Statistics.Percentile(values, 75); +double[] values = ts.ValuesToArray(); +double p25 = Statistics.Percentile(values, 0.25); +double p50 = Statistics.Percentile(values, 0.50); +double p75 = Statistics.Percentile(values, 0.75); Console.WriteLine($"25th percentile: {p25:F1}"); Console.WriteLine($"Median: {p50:F1}"); @@ -254,35 +287,139 @@ Console.WriteLine($"75th percentile: {p75:F1}"); ### Moving Average +A **simple moving average** (SMA) of period $m$ smooths the series by replacing each value with the average of the preceding $m$ observations: + +```math +\text{SMA}_t = \frac{1}{m} \sum_{j=0}^{m-1} x_{t-j} +``` + +The moving average acts as a low-pass filter: it attenuates fluctuations with period shorter than $m$ while preserving longer-term trends. The output series has $n - m + 1$ values because the first $m - 1$ observations lack a full window. + +The ***Numerics*** library provides built-in `MovingAverage` and `MovingSum` methods that use a sliding window for $O(n)$ efficiency: + ```cs -// Compute moving average -int window = 3; -var movingAvg = new TimeSeries(ts.TimeInterval); +var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(2024, 1, 1), + new[] { 125.0, 130.0, 135.0, 132.0, 138.0, 145.0 }); -for (int i = window - 1; i < ts.Count; i++) -{ - double sum = 0; - for (int j = 0; j < window; j++) - { - sum += ts[i - j].Value; - } - double avg = sum / window; - movingAvg.Add(new SeriesOrdinate(ts[i].Index, avg)); -} +// Built-in moving average +var movingAvg = ts.MovingAverage(period: 3); -Console.WriteLine("Original | 3-day Moving Average"); -for (int i = 0; i < ts.Count; i++) +// Built-in moving sum +var movingSum = ts.MovingSum(period: 3); + +Console.WriteLine("Original | 3-day MA | 3-day Sum"); +for (int i = 0; i < movingAvg.Count; i++) { - string ma = i < movingAvg.Count ? $"{movingAvg[i].Value:F1}" : "N/A"; - Console.WriteLine($"{ts[i].Value,8:F1} | {ma,22}"); + Console.WriteLine($"{movingAvg[i].Index:yyyy-MM-dd}: {movingAvg[i].Value,8:F1} | {movingSum[i].Value,8:F1}"); } ``` +## Block Series + +The ***Numerics*** library can aggregate a time series into annual, monthly, or quarterly blocks using a specified function (minimum, maximum, average, or sum). This is essential for extracting annual maxima for flood frequency analysis, computing monthly means, or aggregating sub-daily data. + +### Calendar and Water Year Series + +```cs +// Annual maximum flow (calendar year: Jan–Dec) +var annualMax = dailyFlow.CalendarYearSeries(BlockFunctionType.Maximum); + +// Water year maximum (Oct–Sep, standard in US hydrology) +var waterYearMax = dailyFlow.CustomYearSeries(startMonth: 10, BlockFunctionType.Maximum); + +// Custom season: June–August summer average +var summerAvg = dailyFlow.CustomYearSeries( + startMonth: 6, endMonth: 8, BlockFunctionType.Average); +``` + +### Monthly and Quarterly Series + +```cs +// Monthly average flow +var monthlyAvg = dailyFlow.MonthlySeries(BlockFunctionType.Average); + +// Quarterly maximum +var quarterlyMax = dailyFlow.QuarterlySeries(BlockFunctionType.Maximum); +``` + +### Smoothing Before Aggregation + +Block methods support optional smoothing before aggregation. For example, to find the annual maximum 7-day average flow (a common low-flow statistic): + +```cs +// Annual maximum of 7-day moving average +var annualMax7Day = dailyFlow.CalendarYearSeries( + BlockFunctionType.Maximum, + SmoothingFunctionType.MovingAverage, + period: 7); +``` + +### Peaks Over Threshold + +Extract independent peaks that exceed a threshold, with a minimum separation between events: + +```cs +// Find flood peaks exceeding 500 cfs, at least 7 days apart +var peaks = dailyFlow.PeaksOverThresholdSeries( + threshold: 500.0, minStepsBetweenEvents: 7); +``` + +## Time Interval Conversion + +Convert between time intervals by aggregation (downsampling) or interpolation (upsampling): + +```cs +// Convert daily to monthly (average) +var monthly = dailyFlow.ConvertTimeInterval(TimeInterval.OneMonth, average: true); + +// Convert daily to monthly (sum — e.g., for precipitation) +var monthlySum = dailyPrecip.ConvertTimeInterval(TimeInterval.OneMonth, average: false); +``` + +When downsampling, `average: true` computes the block mean and `average: false` computes the block sum. When upsampling, `average: true` uses linear interpolation and `average: false` disaggregates proportionally. + +## Missing Data Interpolation + +The `InterpolateMissingData` method fills gaps using linear interpolation in date-space, but only when the gap is smaller than a specified maximum: + +```cs +// Fill gaps of up to 3 missing values by linear interpolation +ts.InterpolateMissingData(maxNumberOfMissing: 3); +``` + +This prevents unreliable interpolation across long gaps while filling short data dropouts. + +## Resampling Methods + +### Block Bootstrap + +The **block bootstrap** preserves temporal dependence by resampling contiguous blocks rather than individual observations. Given a block size $b$, the method randomly selects blocks of $b$ consecutive values (with replacement) and concatenates them: + +```cs +// Generate a 1000-step synthetic series preserving 30-day temporal structure +var resampled = ts.ResampleWithBlockBootstrap( + timeSteps: 1000, blockSize: 30, seed: 42); +``` + +Unlike the standard bootstrap (which destroys autocorrelation), the block bootstrap retains the short-range dependence structure within each block. + +### k-Nearest Neighbors + +The **k-nearest neighbors** (k-NN) resampling method generates synthetic time series that preserve the multivariate dependence structure. At each step, it finds the $k$ nearest neighbors of the current state in the historical record (using Euclidean distance on standardized values) and randomly selects one as the next value: + +```cs +// Generate synthetic series using 5-nearest neighbors +var synthetic = ts.ResampleWithKNN( + timeSteps: 500, k: 5, seed: 42); +``` + ## Sorting and Filtering ### Sorting ```cs +using System.ComponentModel; + // Sort by time (ascending or descending) ts.SortByTime(ListSortDirection.Ascending); @@ -314,15 +451,18 @@ Console.WriteLine($"Filtered to {filtered.Count} values in date range"); ### Example 1: Annual Peak Flow Analysis ```cs +using System.Linq; +using Numerics.Data; + // Monthly flow data double[] monthlyFlow = { 125, 135, 180, 220, 250, 280, 260, 230, 190, 150, 130, 120 }; -var flowData = new TimeSeries(TimeInterval.Monthly, new DateTime(2024, 1, 1), monthlyFlow); +var flowData = new TimeSeries(TimeInterval.OneMonth, new DateTime(2024, 1, 1), monthlyFlow); Console.WriteLine("Monthly Streamflow Analysis"); Console.WriteLine("=" + new string('=', 50)); // Find annual peak -double peakFlow = flowData.Values.Max(); +double peakFlow = flowData.ValuesToArray().Max(); int peakMonth = Array.IndexOf(monthlyFlow, peakFlow) + 1; Console.WriteLine($"Peak flow: {peakFlow:F0} cfs"); @@ -340,6 +480,9 @@ Console.WriteLine($" Summer (JJA): {summer.Average():F0} cfs"); ### Example 2: Filling Missing Values ```cs +using System.Linq; +using Numerics.Data; + // Time series with gaps var dates = new[] { new DateTime(2024, 1, 1), @@ -367,7 +510,7 @@ for (int i = 0; i < ts.Count - 1; i++) } // Fill gaps with linear interpolation -var filled = new TimeSeries(TimeInterval.Daily, dates.Min(), dates.Max()); +var filled = new TimeSeries(TimeInterval.OneDay, dates.Min(), dates.Max()); foreach (var ord in filled) { // Find surrounding values @@ -392,10 +535,14 @@ Console.WriteLine($"\nFilled series: {filled.Count} continuous daily values"); ### Example 3: Trend Detection ```cs +using System.Linq; +using Numerics.Data; +using Numerics.Data.Statistics; + double[] annualPeaks = { 1200, 1250, 1180, 1300, 1320, 1280, 1350, 1400, 1380, 1450 }; var years = Enumerable.Range(2015, 10).Select(y => new DateTime(y, 1, 1)).ToArray(); -var peakSeries = new TimeSeries(TimeInterval.Annual); +var peakSeries = new TimeSeries(TimeInterval.OneYear); for (int i = 0; i < years.Length; i++) { peakSeries.Add(new SeriesOrdinate(years[i], annualPeaks[i])); @@ -404,42 +551,76 @@ for (int i = 0; i < years.Length; i++) Console.WriteLine("Annual Peak Flow Trend Analysis"); Console.WriteLine("=" + new string('=', 50)); -// Linear regression for trend -double[] x = Enumerable.Range(0, peakSeries.Count).Select(i => (double)i).ToArray(); -double[] y = peakSeries.Values.ToArray(); - -double xMean = x.Average(); -double yMean = y.Average(); +// Trend analysis using built-in hypothesis tests +double[] indices = Enumerable.Range(0, peakSeries.Count).Select(i => (double)i).ToArray(); +double[] y = peakSeries.ValuesToArray(); -double slope = x.Zip(y, (xi, yi) => (xi - xMean) * (yi - yMean)).Sum() / - x.Sum(xi => Math.Pow(xi - xMean, 2)); -double intercept = yMean - slope * xMean; +// Parametric: Linear regression t-test (returns p-value) +double linearPValue = HypothesisTests.LinearTrendTest(indices, y); +Console.WriteLine($"Linear trend test p-value: {linearPValue:F4}"); -Console.WriteLine($"Trend: {slope:F1} cfs/year"); -Console.WriteLine($"Direction: {(slope > 0 ? "Increasing" : "Decreasing")}"); +// Non-parametric: Mann-Kendall test (returns p-value) +double mkPValue = HypothesisTests.MannKendallTest(y); +Console.WriteLine($"Mann-Kendall p-value: {mkPValue:F4}"); -// Mann-Kendall test for significance -double mkStat = HypothesisTests.MannKendallTest(y); -Console.WriteLine($"Mann-Kendall statistic: {mkStat:F2}"); - -if (Math.Abs(mkStat) > 1.96) +if (linearPValue < 0.05 || mkPValue < 0.05) Console.WriteLine("Trend is statistically significant (p < 0.05)"); else Console.WriteLine("Trend is not statistically significant"); ``` -### Example 4: Seasonal Analysis +### Example 4: Seasonal Decomposition + +A time series can be decomposed into trend ($T_t$), seasonal ($S_t$), and residual ($R_t$) components. The two standard models are: + +- **Additive**: $x_t = T_t + S_t + R_t$ — when seasonal fluctuations are roughly constant in magnitude +- **Multiplicative**: $x_t = T_t \cdot S_t \cdot R_t$ — when seasonal fluctuations scale with the level + +The `SeasonalDecompose` method performs classical additive decomposition using a moving average for the trend and FFT-based extraction for the seasonal component: + +```cs +using System.Linq; +using Numerics.Data; + +// Monthly temperature data (5 years) +int nYears = 5; +int period = 12; +var random = new Random(123); +var monthlyTemp = new TimeSeries(TimeInterval.OneMonth); + +for (int i = 0; i < nYears * period; i++) +{ + double trend = 15.0 + 0.05 * i; // Slight warming trend + double seasonal = 10.0 * Math.Sin(2 * Math.PI * i / period); + double noise = (random.NextDouble() - 0.5) * 2; + monthlyTemp.Add(new SeriesOrdinate( + new DateTime(2020, 1, 1).AddMonths(i), trend + seasonal + noise)); +} + +// Decompose into trend, seasonal, and residual components +var (trend, seasonal, residual) = monthlyTemp.SeasonalDecompose(period); + +Console.WriteLine("Seasonal Decomposition:"); +Console.WriteLine($" Trend range: {trend.MinValue():F1} to {trend.MaxValue():F1}"); +Console.WriteLine($" Seasonal amplitude: {seasonal.Max() - seasonal.Min():F1}"); +Console.WriteLine($" Residual count: {residual.Count}"); +``` + +### Example 5: Seasonal Analysis ```cs +using System.Linq; +using Numerics.Data; + // Multi-year daily data -int years = 3; +int nYears = 3; int daysPerYear = 365; var random = new Random(123); -var dailyTemp = new TimeSeries(TimeInterval.Daily, new DateTime(2022, 1, 1)); +var dailyTemp = new TimeSeries(TimeInterval.OneDay); // Generate seasonal temperature pattern -for (int day = 0; day < years * daysPerYear; day++) +for (int day = 0; day < nYears * daysPerYear; day++) { // Sinusoidal pattern + noise double seasonalTemp = 15 + 10 * Math.Sin(2 * Math.PI * day / 365.0); @@ -453,22 +634,14 @@ for (int day = 0; day < years * daysPerYear; day++) Console.WriteLine("Seasonal Temperature Analysis"); Console.WriteLine("=" + new string('=', 50)); -// Compute monthly averages -var monthlyAvg = new Dictionary>(); -for (int m = 1; m <= 12; m++) - monthlyAvg[m] = new List(); - -foreach (var ord in dailyTemp) -{ - monthlyAvg[ord.Index.Month].Add(ord.Value); -} +// Compute monthly averages using built-in MonthlySeries +var monthlyAvgSeries = dailyTemp.MonthlySeries(BlockFunctionType.Average); -Console.WriteLine("\nMonth | Avg Temp (°C)"); -Console.WriteLine("------|-------------"); -for (int m = 1; m <= 12; m++) +Console.WriteLine("\nDate | Avg Temp (°C)"); +Console.WriteLine("-----------|-------------"); +foreach (var ord in monthlyAvgSeries) { - double avg = monthlyAvg[m].Average(); - Console.WriteLine($"{m,5} | {avg,13:F1}"); + Console.WriteLine($"{ord.Index:yyyy-MM} | {ord.Value,13:F1}"); } ``` @@ -490,9 +663,24 @@ for (int m = 1; m <= 12; m++) | Standardize | `Standardize()` | Compare different scales | | Cumulative | `CumulativeSum()` | Total accumulation | | Difference | `Difference(lag)` | Remove trends | -| Moving average | Custom loop | Smoothing | +| Moving average | `MovingAverage(period)` | Smoothing | +| Moving sum | `MovingSum(period)` | Accumulation over window | +| Annual block | `CalendarYearSeries()` | Annual statistics | +| Monthly block | `MonthlySeries()` | Monthly aggregation | +| Convert interval | `ConvertTimeInterval()` | Up/downsampling | +| Peaks over threshold | `PeaksOverThresholdSeries()` | Event extraction | +| Block bootstrap | `ResampleWithBlockBootstrap()` | Synthetic generation | +| k-NN resampling | `ResampleWithKNN()` | Synthetic generation | | Sort | `SortByTime()` | Ensure chronological order | --- -[← Previous: Interpolation](interpolation.md) | [Back to Index](../index.md) | [Next: Random Generation →](../sampling/random-generation.md) +## References + +[1] Box, G. E. P., Jenkins, G. M., Reinsel, G. C., & Ljung, G. M. (2015). *Time Series Analysis: Forecasting and Control* (5th ed.). Wiley. + +[2] Kundzewicz, Z. W. & Robson, A. J. (2004). Change detection in hydrological records — a review of the methodology. *Hydrological Sciences Journal*, 49(1), 7–19. + +--- + +[← Previous: Linear Regression](regression.md) | [Back to Index](../index.md) | [Next: Descriptive Statistics →](../statistics/descriptive.md) diff --git a/docs/distributions/copulas.md b/docs/distributions/copulas.md index 6b464d7c..51c42926 100644 --- a/docs/distributions/copulas.md +++ b/docs/distributions/copulas.md @@ -1,75 +1,246 @@ # Copulas -[← Previous: Uncertainty Analysis](uncertainty-analysis.md) | [Back to Index](../index.md) +[← Previous: Uncertainty Analysis](uncertainty-analysis.md) | [Back to Index](../index.md) | [Next: Multivariate Distributions →](multivariate.md) -Copulas separate the dependence structure of multivariate distributions from their marginal distributions. The ***Numerics*** library provides copula functions for modeling dependence between random variables in risk assessment and multivariate analysis [[1]](#1). +Copulas separate the dependence structure of multivariate distributions from their marginal distributions. This separation is formalized by **Sklar's theorem** [[1]](#1): for any multivariate distribution $F$ with marginal CDFs $F_1, F_2, \ldots, F_n$, there exists a copula $C$ such that: -## Overview - -A copula is a multivariate distribution with uniform marginals on [0,1]. For any multivariate distribution with marginals F₁, F₂, ..., Fₙ, there exists a copula C such that: - -``` -F(x₁, x₂, ..., xₙ) = C(F₁(x₁), F₂(x₂), ..., Fₙ(xₙ)) +```math +F(x_1, x_2, \ldots, x_n) = C(F_1(x_1), F_2(x_2), \ldots, F_n(x_n)) ``` -This allows us to: -1. Model marginal distributions independently -2. Model dependence separately via copula -3. Combine them to form joint distribution +A copula $C: [0,1]^n \rightarrow [0,1]$ is itself a multivariate CDF with uniform marginals. The power of this decomposition is that it allows us to: +1. Model marginal distributions independently (each can be any distribution) +2. Model dependence separately via the copula +3. Combine them to form any joint distribution + +The ***Numerics*** library provides bivariate copula functions for modeling dependence between two random variables in risk assessment and multivariate analysis. ## Available Copulas -The library provides common copula families: +### Elliptical Copulas + +Elliptical copulas are derived from elliptical distributions (Normal, Student's t). They produce symmetric dependence structures. + +#### Normal (Gaussian) Copula + +The Normal copula is derived from the bivariate normal distribution. Its density function is: + +```math +c(u, v) = \frac{1}{\sqrt{1-\rho^2}} \exp\left(-\frac{\rho^2 s^2 + \rho^2 t^2 - 2\rho s t}{2(1-\rho^2)}\right) +``` + +where $s = \Phi^{-1}(u)$ and $t = \Phi^{-1}(v)$ are the standard normal quantiles, and $\rho \in [-1, 1]$ is the correlation parameter. + +The Normal copula has **no tail dependence** ($\lambda_L = \lambda_U = 0$), meaning that extreme events in one variable do not increase the probability of extreme events in the other beyond what the overall correlation implies. This makes it unsuitable for modeling joint extremes. -### Gaussian Copula +The relationship between Kendall's tau and the copula parameter is: -Models linear correlation with normal dependence structure: +```math +\tau = \frac{2}{\pi} \arcsin(\rho) +``` ```cs using Numerics.Distributions.Copulas; -// Correlation matrix for 2D Gaussian copula +// Normal (Gaussian) copula with correlation rho double rho = 0.7; // Correlation coefficient -var corrMatrix = new double[,] { { 1.0, rho }, { rho, 1.0 } }; -var gaussianCopula = new GaussianCopula(corrMatrix); +var normalCopula = new NormalCopula(rho); // Evaluate copula density double u1 = 0.3, u2 = 0.7; -double density = gaussianCopula.PDF(new double[] { u1, u2 }); +double density = normalCopula.PDF(u1, u2); + +Console.WriteLine($"Normal copula density: {density:F4}"); +``` -Console.WriteLine($"Gaussian copula density: {density:F4}"); +#### Student's t Copula + +The Student's t copula extends the Normal copula by adding **symmetric tail dependence**. Its density involves the bivariate Student's t distribution with $\nu$ degrees of freedom and correlation $\rho$. The density is computed in log-space for numerical stability: + +```math +\log c(u,v) = \log \Gamma\!\left(\frac{\nu+2}{2}\right) + \log \Gamma\!\left(\frac{\nu}{2}\right) - 2\log \Gamma\!\left(\frac{\nu+1}{2}\right) - \frac{1}{2}\log(1-\rho^2) - \frac{\nu+2}{2}\log\!\left(1 + \frac{Q}{\nu(1-\rho^2)}\right) + \frac{\nu+1}{2}\left[\log\!\left(1 + \frac{x_1^2}{\nu}\right) + \log\!\left(1 + \frac{x_2^2}{\nu}\right)\right] ``` -### Student's t Copula +where $x_1 = t_\nu^{-1}(u)$, $x_2 = t_\nu^{-1}(v)$, and $Q = x_1^2 - 2\rho x_1 x_2 + x_2^2$. -Similar to Gaussian but with tail dependence: +The **tail dependence coefficient** is: + +```math +\lambda = \lambda_L = \lambda_U = 2 \cdot t_{\nu+1}\!\left(-\sqrt{\frac{(\nu+1)(1-\rho)}{1+\rho}}\right) +``` + +As $\nu \to \infty$, the t copula converges to the Normal copula and tail dependence vanishes. As $\nu$ decreases, tail dependence increases. ```cs // t-copula with 5 degrees of freedom int nu = 5; -var tCopula = new StudentTCopula(corrMatrix, nu); +var tCopula = new StudentTCopula(rho, nu); -double density = tCopula.PDF(new double[] { u1, u2 }); +double density = tCopula.PDF(u1, u2); Console.WriteLine($"t-copula density: {density:F4}"); -Console.WriteLine("t-copula has stronger tail dependence than Gaussian"); +Console.WriteLine($"Upper tail dependence: {tCopula.UpperTailDependence:F4}"); +Console.WriteLine($"Lower tail dependence: {tCopula.LowerTailDependence:F4}"); ``` ### Archimedean Copulas -Family of copulas with specific dependence structures: +Archimedean copulas are defined through a **generator function** $\varphi(t)$, a continuous, strictly decreasing, convex function with $\varphi(1) = 0$. The copula CDF is expressed as: + +```math +C(u, v) = \varphi^{-1}(\varphi(u) + \varphi(v)) +``` + +The copula density is obtained by differentiating: + +```math +c(u,v) = -\frac{\varphi''(\varphi^{-1}(\varphi(u) + \varphi(v)))}{[\varphi'(\varphi^{-1}(\varphi(u) + \varphi(v)))]^3} \cdot \varphi'(u) \cdot \varphi'(v) +``` + +Each Archimedean family is characterized by its generator, which determines the copula's dependence properties. The ***Numerics*** library implements five Archimedean families: + +#### Clayton Copula + +The Clayton copula exhibits **lower tail dependence**, making it suitable for modeling the joint occurrence of low values (e.g., concurrent droughts in multiple watersheds). + +| Property | Value | +|----------|-------| +| Generator | $\varphi(t) = t^{-\theta} - 1$ | +| Inverse generator | $\varphi^{-1}(s) = (1+s)^{-1/\theta}$ | +| CDF | $C(u,v) = (u^{-\theta} + v^{-\theta} - 1)^{-1/\theta}$ | +| Parameter range | $\theta \in (0, \infty)$ | +| Kendall's tau | $\tau = \frac{\theta}{\theta + 2}$ | +| Lower tail dependence | $\lambda_L = 2^{-1/\theta}$ | +| Upper tail dependence | $\lambda_U = 0$ | + +```cs +// Clayton copula (lower tail dependence), θ ∈ (0, ∞) +var claytonCopula = new ClaytonCopula(2.0); +``` + +#### Gumbel Copula + +The Gumbel copula exhibits **upper tail dependence**, making it the natural choice for modeling joint flood events (e.g., concurrent high flows in tributaries). + +| Property | Value | +|----------|-------| +| Generator | $\varphi(t) = (-\ln t)^{\theta}$ | +| Inverse generator | $\varphi^{-1}(s) = \exp(-s^{1/\theta})$ | +| Parameter range | $\theta \in [1, \infty)$ | +| Kendall's tau | $\tau = 1 - \frac{1}{\theta}$ | +| Lower tail dependence | $\lambda_L = 0$ | +| Upper tail dependence | $\lambda_U = 2 - 2^{1/\theta}$ | + +```cs +// Gumbel copula (upper tail dependence), θ ∈ [1, ∞) +var gumbelCopula = new GumbelCopula(2.0); +``` + +#### Frank Copula + +The Frank copula has **no tail dependence** and produces a symmetric dependence structure. It is useful for modeling moderate, general correlation without emphasizing joint extremes. + +| Property | Value | +|----------|-------| +| Generator | $\varphi(t) = -\ln\!\left(\frac{e^{-\theta t} - 1}{e^{-\theta} - 1}\right)$ | +| CDF | $C(u,v) = -\frac{1}{\theta}\ln\!\left(1 + \frac{(e^{-\theta u}-1)(e^{-\theta v}-1)}{e^{-\theta}-1}\right)$ | +| Parameter range | $\theta \in (-\infty, \infty) \setminus \lbrace 0\rbrace$ | +| Tail dependence | $\lambda_L = \lambda_U = 0$ | + +The Frank copula is the only Archimedean copula that allows both positive and negative dependence ($\theta > 0$ for positive, $\theta < 0$ for negative). + +```cs +// Frank copula (no tail dependence), θ ∈ (-∞, ∞) \ {0} +var frankCopula = new FrankCopula(5.0); +``` + +#### Joe Copula + +The Joe copula has **upper tail dependence**, similar to the Gumbel copula, but with a different dependence structure in the body of the distribution. + +| Property | Value | +|----------|-------| +| Generator | $\varphi(t) = -\ln(1 - (1-t)^{\theta})$ | +| Parameter range | $\theta \in [1, \infty)$ | +| Lower tail dependence | $\lambda_L = 0$ | +| Upper tail dependence | $\lambda_U = 2 - 2^{1/\theta}$ | + +```cs +// Joe copula (upper tail dependence), θ ∈ [1, ∞) +var joeCopula = new JoeCopula(2.0); +``` + +#### Ali-Mikhail-Haq (AMH) Copula + +The AMH copula models **weak dependence structures** and has no tail dependence. Its parameter range is limited to $[-1, 1]$, which restricts it to Kendall's tau values in approximately $[-0.182, 0.333]$. + +| Property | Value | +|----------|-------| +| Generator | $\varphi(t) = \ln\!\left(\frac{1-\theta(1-t)}{t}\right)$ | +| Parameter range | $\theta \in [-1, 1]$ | +| Kendall's tau range | $\tau \in \left[\frac{5 - 8\ln 2}{3}, \frac{1}{3}\right] \approx [-0.182, 0.333]$ | +| Tail dependence | $\lambda_L = \lambda_U = 0$ | + +```cs +// Ali-Mikhail-Haq copula (weak dependence), θ ∈ [-1, 1] +var amhCopula = new AMHCopula(0.5); +``` + +### Copula Selection Guide + +| Copula | Tail Dependence | Parameter Range | Best For | +|--------|----------------|-----------------|----------| +| Normal | None | $\rho \in [-1, 1]$ | General symmetric dependence | +| Student-t | Symmetric | $\rho \in [-1, 1]$, $\nu > 2$ | Heavy-tailed joint extremes | +| Clayton | Lower tail | $\theta \in (0, \infty)$ | Joint low extremes (droughts) | +| Gumbel | Upper tail | $\theta \in [1, \infty)$ | Joint high extremes (floods) | +| Frank | None | $\theta \in \mathbb{R} \setminus \lbrace 0\rbrace$ | Moderate symmetric dependence | +| Joe | Upper tail | $\theta \in [1, \infty)$ | Strong upper tail dependence | +| AMH | None | $\theta \in [-1, 1]$ | Weak dependence structures | + +## Fitting Copulas to Data + +The copula parameter can be estimated from data using the relationship between Kendall's tau and the copula parameter. The ***Numerics*** library provides the `SetThetaFromTau` method for Archimedean copulas: ```cs -// Clayton copula (lower tail dependence) -double theta = 2.0; // Dependence parameter -var claytonCopula = new ClaytonCopula(theta); +using System.Linq; +using Numerics.Data.Statistics; +using Numerics.Distributions; +using Numerics.Distributions.Copulas; + +// Sample paired observations (e.g., peak flow and volume) +double[] x = { 1200, 1500, 1100, 1800, 1350, 1600, 1250, 1450, 1900, 1300 }; +double[] y = { 45, 52, 42, 65, 48, 58, 44, 51, 68, 46 }; + +// Step 1: Fit marginal distributions +var gevX = new GeneralizedExtremeValue(); +gevX.Estimate(x, ParameterEstimationMethod.MethodOfLinearMoments); + +var gevY = new GeneralizedExtremeValue(); +gevY.Estimate(y, ParameterEstimationMethod.MethodOfLinearMoments); -// Gumbel copula (upper tail dependence) -var gumbelCopula = new GumbelCopula(theta); +// Step 2: Transform to uniform margins using fitted CDFs +double[] u = x.Select(xi => gevX.CDF(xi)).ToArray(); +double[] v = y.Select(yi => gevY.CDF(yi)).ToArray(); -// Frank copula (no tail dependence) -var frankCopula = new FrankCopula(theta); +// Step 3: Estimate copula parameter using rank correlation +double tau = Correlation.KendallsTau(x, y); +Console.WriteLine($"Kendall's tau: {tau:F3}"); + +// The tau-to-parameter relationships: +// Clayton: θ = 2τ/(1-τ) for τ > 0 +// Gumbel: θ = 1/(1-τ) for τ > 0 +// Normal: ρ = sin(πτ/2) +``` + +Alternatively, copula parameters can be estimated by maximizing the pseudo-log-likelihood, which uses only the copula density evaluated at empirical probability-integral transforms: + +```cs +// The BivariateCopula base class provides likelihood methods: +// PseudoLogLikelihood — copula-only log-likelihood +// IFMLogLikelihood — inference functions for margins (with pre-estimated marginals) +// LogLikelihood — full log-likelihood (copula + marginals) ``` ## Practical Example: Bivariate Distribution @@ -77,6 +248,8 @@ var frankCopula = new FrankCopula(theta); Construct a bivariate distribution with arbitrary marginals and specified dependence: ```cs +using System.Linq; +using Numerics.Data.Statistics; using Numerics.Distributions; using Numerics.Distributions.Copulas; @@ -86,12 +259,11 @@ var margin2 = new Gumbel(100, 20); // Peak stage // Step 2: Define dependence via copula double rho = 0.8; // Strong positive correlation -var corrMatrix = new double[,] { { 1.0, rho }, { rho, 1.0 } }; -var copula = new GaussianCopula(corrMatrix); +var copula = new NormalCopula(rho); // Step 3: Sample from joint distribution int n = 1000; -var samples = copula.Sample(n); +var samples = copula.GenerateRandomValues(n); // Transform uniforms to actual distributions double[] flow = new double[n]; @@ -112,6 +284,8 @@ var correlation = Correlation.Pearson(flow, stage); Console.WriteLine($"Sample correlation: {correlation:F3}"); ``` +The sampling algorithm uses the **conditional method**: generate $u$ from $U(0,1)$, then generate $v$ from the conditional distribution $C(v|u) = \frac{\partial C(u,v)}{\partial u}$ by inverting this conditional CDF. The `GenerateRandomValues` method uses Latin Hypercube sampling for better coverage of the probability space. + ## Applications in Risk Assessment ### Joint Probability Analysis @@ -128,119 +302,103 @@ double u1 = margin1.CDF(flowThreshold); double u2 = margin2.CDF(stageThreshold); // P(Flow > threshold AND Stage > threshold) -double jointExceedance = copula.Survival(new double[] { u1, u2 }); +// = 1 - u1 - u2 + C(u1, u2) +double jointExceedance = copula.ANDJointExceedanceProbability(u1, u2); Console.WriteLine($"Joint exceedance probability: {jointExceedance:E4}"); Console.WriteLine($"Return period: {1.0 / jointExceedance:F1} years"); ``` +The AND joint exceedance probability is computed as $P(X > x \text{ and } Y > y) = 1 - u - v + C(u, v)$, while the OR joint exceedance is $P(X > x \text{ or } Y > y) = 1 - C(u, v)$. + ### Conditional Distributions -Given flow, what is the conditional distribution of stage? +Given flow, what is the conditional distribution of stage? The conditional CDF can be computed numerically using the copula CDF via partial differentiation: $C(v|u) = \frac{\partial C(u,v)}{\partial u}$. ```cs // Observed flow double observedFlow = 12000; double uFlow = margin1.CDF(observedFlow); -// Conditional CDF for stage | flow +// Approximate the conditional CDF: dC(u,v)/du via finite difference Func conditionalCDF = (stage) => { double uStage = margin2.CDF(stage); - return copula.ConditionalCDF(uFlow, uStage, conditionIndex: 0); + double du = 1e-6; + double uPlus = Math.Min(uFlow + du, 1.0); + double uMinus = Math.Max(uFlow - du, 0.0); + return (copula.CDF(uPlus, uStage) - copula.CDF(uMinus, uStage)) / (uPlus - uMinus); }; -// Find conditional quantiles -double[] condProbs = { 0.5, 0.9, 0.95 }; +// Conditional probability at specific values Console.WriteLine($"Given flow = {observedFlow:F0} cfs:"); -foreach (var p in condProbs) -{ - // This would require numerical solution - Console.WriteLine($" {p:P0} quantile of stage"); -} +Console.WriteLine($" P(Stage > 15 | Flow = {observedFlow}) = {1 - conditionalCDF(15):P1}"); ``` ## Tail Dependence -Different copulas have different tail dependence properties: +Tail dependence measures the probability of observing an extreme value in one variable given that the other is already extreme. The **upper tail dependence coefficient** is: -```cs -// Gaussian: No tail dependence (λ_L = λ_U = 0) -// t-copula: Symmetric tail dependence -// Clayton: Lower tail dependence only -// Gumbel: Upper tail dependence only -// Frank: No tail dependence +```math +\lambda_U = \lim_{u \to 1^-} P(Y > F_Y^{-1}(u) \mid X > F_X^{-1}(u)) = \lim_{u \to 1^-} \frac{1 - 2u + C(u,u)}{1-u} +``` -Console.WriteLine("Tail Dependence Properties:"); -Console.WriteLine(" Gaussian: No tail dependence"); -Console.WriteLine(" Student-t: Symmetric tail dependence"); -Console.WriteLine(" Clayton: Lower tail dependence (joint lows)"); -Console.WriteLine(" Gumbel: Upper tail dependence (joint highs)"); -Console.WriteLine(" Frank: No tail dependence"); +The **lower tail dependence coefficient** is defined analogously as $u \to 0^+$: -// For flood analysis: Gumbel copula captures joint extremes -// For drought analysis: Clayton copula captures joint lows +```math +\lambda_L = \lim_{u \to 0^+} \frac{C(u,u)}{u} ``` -## Fitting Copulas to Data +A copula has tail dependence if $\lambda > 0$. This is a critical property for risk analysis: the Normal copula always has $\lambda = 0$, meaning it systematically underestimates joint extreme events. For flood analysis, copulas with upper tail dependence (Gumbel, Joe, Student-t) are preferred. ```cs -double[] x = { /* observed data series 1 */ }; -double[] y = { /* observed data series 2 */ }; - -// Step 1: Transform to uniform margins (pseudo-observations) -var u = x.Select(xi => (double)Array.FindIndex(x.OrderBy(v => v).ToArray(), v => v == xi) / x.Length); -var v = y.Select(yi => (double)Array.FindIndex(y.OrderBy(w => w).ToArray(), w => w == yi) / y.Length); - -// Step 2: Fit copula to pseudo-observations -// Use maximum likelihood or rank correlation methods - -// Step 3: Select best copula using AIC/BIC -var candidates = new[] { "Gaussian", "t", "Clayton", "Gumbel", "Frank" }; - -Console.WriteLine("Fit each candidate copula and select best by AIC"); +// Tail dependence comparison using UpperTailDependence / LowerTailDependence properties +Console.WriteLine("Tail Dependence Properties:"); +Console.WriteLine($" Student-t(ρ=0.7, ν=5): λ_U = {new StudentTCopula(0.7, 5).UpperTailDependence:F4}"); +Console.WriteLine($" Student-t(ρ=0.7, ν=20): λ_U = {new StudentTCopula(0.7, 20).UpperTailDependence:F4}"); +Console.WriteLine($" Normal(ρ=0.7): λ_U = {new NormalCopula(0.7).UpperTailDependence:F4}"); +Console.WriteLine($" Clayton(θ=2): λ_L = {new ClaytonCopula(2.0).LowerTailDependence:F4}"); +Console.WriteLine($" Gumbel(θ=2): λ_U = {new GumbelCopula(2.0).UpperTailDependence:F4}"); +Console.WriteLine($" Frank(θ=5): λ_U = {new FrankCopula(5.0).UpperTailDependence:F4}"); ``` -## Vine Copulas +## Higher-Dimensional Dependence -For higher dimensions, vine copulas decompose multivariate dependence: +For problems with more than two variables, there are several approaches: -```cs -// C-vine, D-vine, and R-vine structures available -// Allow flexible modeling of high-dimensional dependence -// Construct from pairwise bivariate copulas +1. **Multivariate Normal distribution**: Use when Gaussian dependence is appropriate +2. **Nested Archimedean copulas**: Hierarchical structure for grouped variables +3. **Vine copulas**: Build from pairwise bivariate copulas (C-vine, D-vine, R-vine) -Console.WriteLine("Vine copulas enable flexible high-dimensional modeling"); -Console.WriteLine("Use for systems with > 2 correlated variables"); -``` +For hydrologic applications with 3+ correlated variables (e.g., peak flow, volume, duration), consider using the Multivariate Normal distribution for the initial analysis, which is available in the `Numerics.Distributions` namespace. See the [Multivariate Distributions](multivariate.md) documentation for details. ## Best Practices -1. **Check for dependence**: Use scatter plots and correlation tests before applying copulas -2. **Choose copula family**: Match tail behavior to application - - Joint extremes → Gumbel - - Joint lows → Clayton - - Moderate correlation → Gaussian - - Heavy tails → Student-t -3. **Validate fit**: Check if copula captures observed dependence structure -4. **Sample size**: Need sufficient data (n > 50-100) for reliable fitting -5. **Non-stationarity**: Check if dependence structure is time-varying +1. **Check for dependence**: Use scatter plots and correlation tests before applying copulas. Kendall's tau is preferred over Pearson's $r$ because it is rank-based and invariant under monotonic transformations +2. **Choose copula family**: Match tail behavior to application — Gumbel for joint highs, Clayton for joint lows, Student-t for symmetric tail dependence +3. **Validate fit**: Compare empirical and theoretical joint exceedance curves. Use the pseudo-log-likelihood to compare copula families +4. **Sample size**: Need sufficient data ($n > 50$–$100$) for reliable parameter estimation via Kendall's tau +5. **Non-stationarity**: Check if dependence structure is time-varying — a copula fit to historical data may not represent future conditions ## Limitations -- Assumes marginals are correctly specified -- May not capture complex nonlinear dependencies -- Parameter estimation challenging with limited data -- Tail dependence difficult to estimate from finite samples +- Assumes marginals are correctly specified — copula inference is only valid when the marginal distributions are well-fitted +- Bivariate copulas may not capture complex nonlinear dependencies in higher dimensions +- Parameter estimation is challenging with limited data, especially for tail dependence +- Tail dependence is difficult to estimate from finite samples — the tail region by definition has few observations --- ## References -[1] Nelsen, R. B. (2006). *An Introduction to Copulas* (2nd ed.). Springer. +[1] R. B. Nelsen, *An Introduction to Copulas*, 2nd ed., New York: Springer, 2006. + +[2] H. Joe, *Dependence Modeling with Copulas*, Boca Raton: CRC Press, 2014. + +[3] C. Genest and A.-C. Favre, "Everything you always wanted to know about copula modeling but were afraid to ask," *Journal of Hydrologic Engineering*, vol. 12, no. 4, pp. 347-368, 2007. -[2] Joe, H. (2014). *Dependence Modeling with Copulas*. CRC Press. +[4] G. Salvadori, C. De Michele, N. T. Kottegoda and R. Rosso, *Extremes in Nature: An Approach Using Copulas*, Dordrecht: Springer, 2007. --- -[← Previous: Uncertainty Analysis](uncertainty-analysis.md) | [Back to Index](../index.md) +[← Previous: Uncertainty Analysis](uncertainty-analysis.md) | [Back to Index](../index.md) | [Next: Multivariate Distributions →](multivariate.md) diff --git a/docs/distributions/multivariate.md b/docs/distributions/multivariate.md index 650b6d2b..0125ec55 100644 --- a/docs/distributions/multivariate.md +++ b/docs/distributions/multivariate.md @@ -1,13 +1,72 @@ # Multivariate Distributions -[← Previous: Copulas](copulas.md) | [Back to Index](../index.md) +[← Previous: Copulas](copulas.md) | [Back to Index](../index.md) | [Next: Machine Learning →](../machine-learning/machine-learning.md) -The ***Numerics*** library provides the **Multivariate Normal** distribution for modeling correlated random variables. This distribution is fundamental in multivariate statistics, risk assessment, and uncertainty quantification. +The ***Numerics*** library provides several multivariate distributions for modeling correlated random variables: the **Multivariate Normal**, **Multivariate Student-t**, **Dirichlet**, and **Multinomial** distributions. These are fundamental in multivariate statistics, risk assessment, and uncertainty quantification. ## Multivariate Normal Distribution The multivariate normal (Gaussian) distribution generalizes the univariate normal to multiple dimensions with a specified covariance structure [[1]](#1). +### Mathematical Definition + +A random vector $\mathbf{X} = (X_1, X_2, \ldots, X_k)^T$ follows a $k$-dimensional multivariate normal distribution, written $\mathbf{X} \sim \mathcal{N}_k(\boldsymbol{\mu}, \boldsymbol{\Sigma})$, if its probability density function is: + +```math +f(\mathbf{x}) = (2\pi)^{-k/2} \, |\boldsymbol{\Sigma}|^{-1/2} \, \exp\!\left( -\tfrac{1}{2} (\mathbf{x} - \boldsymbol{\mu})^T \boldsymbol{\Sigma}^{-1} (\mathbf{x} - \boldsymbol{\mu}) \right) +``` + +where: +- $\boldsymbol{\mu} \in \mathbb{R}^k$ is the mean vector, +- $\boldsymbol{\Sigma}$ is the $k \times k$ positive-definite covariance matrix, +- $|\boldsymbol{\Sigma}|$ is the determinant of $\boldsymbol{\Sigma}$. + +**Mahalanobis distance.** The quadratic form in the exponent defines the squared Mahalanobis distance between a point $\mathbf{x}$ and the distribution center $\boldsymbol{\mu}$: + +```math +D^2(\mathbf{x}) = (\mathbf{x} - \boldsymbol{\mu})^T \boldsymbol{\Sigma}^{-1} (\mathbf{x} - \boldsymbol{\mu}) +``` + +The PDF can be expressed compactly in terms of this distance as $`f(\mathbf{x}) = (2\pi)^{-k/2} |\boldsymbol{\Sigma}|^{-1/2} \exp(-D^2/2)`$. Surfaces of constant density are ellipsoids defined by $D^2 = c$, and the squared Mahalanobis distance follows a chi-squared distribution: $D^2 \sim \chi^2_k$. This property is used for multivariate outlier detection -- a point is flagged as an outlier if $D^2$ exceeds the $\chi^2_k$ critical value at the desired significance level. + +**Marginal distributions.** Any subset of variables from a multivariate normal is itself multivariate normal. If the full vector is partitioned as $\mathbf{X} = (\mathbf{X}_a, \mathbf{X}_b)^T$ with corresponding partitioned mean and covariance: + +```math +\boldsymbol{\mu} = \begin{pmatrix} \boldsymbol{\mu}_a \\ \boldsymbol{\mu}_b \end{pmatrix}, \quad \boldsymbol{\Sigma} = \begin{pmatrix} \boldsymbol{\Sigma}_{aa} & \boldsymbol{\Sigma}_{ab} \\ \boldsymbol{\Sigma}_{ba} & \boldsymbol{\Sigma}_{bb} \end{pmatrix} +``` + +then the marginal distribution is $`\mathbf{X}_a \sim \mathcal{N}(\boldsymbol{\mu}_a, \boldsymbol{\Sigma}_{aa})`$. + +**Conditional distributions.** The conditional distribution of $\mathbf{X}_a$ given $\mathbf{X}_b = \mathbf{x}_b$ is also multivariate normal: + +```math +\mathbf{X}_a \mid \mathbf{X}_b = \mathbf{x}_b \sim \mathcal{N}\!\left( \boldsymbol{\mu}_{a|b}, \, \boldsymbol{\Sigma}_{a|b} \right) +``` + +with conditional mean and covariance: + +```math +\boldsymbol{\mu}_{a|b} = \boldsymbol{\mu}_a + \boldsymbol{\Sigma}_{ab} \boldsymbol{\Sigma}_{bb}^{-1} (\mathbf{x}_b - \boldsymbol{\mu}_b) +``` + +```math +\boldsymbol{\Sigma}_{a|b} = \boldsymbol{\Sigma}_{aa} - \boldsymbol{\Sigma}_{ab} \boldsymbol{\Sigma}_{bb}^{-1} \boldsymbol{\Sigma}_{ba} +``` + +Note that the conditional covariance $\boldsymbol{\Sigma}_{a|b}$ does not depend on the observed value $\mathbf{x}_b$. For the bivariate case ($k = 2$), these reduce to: + +```math +\mu_{Y|X=x} = \mu_Y + \rho \frac{\sigma_Y}{\sigma_X}(x - \mu_X), \quad \sigma^2_{Y|X} = \sigma^2_Y (1 - \rho^2) +``` + +**Cholesky sampling.** Random samples are generated via the Cholesky decomposition $\boldsymbol{\Sigma} = \mathbf{L}\mathbf{L}^T$, where $\mathbf{L}$ is a lower-triangular matrix. Given a vector of independent standard normal variates $\mathbf{z} \sim \mathcal{N}(\mathbf{0}, \mathbf{I}_k)$: + +```math +\mathbf{x} = \boldsymbol{\mu} + \mathbf{L}\mathbf{z} +``` + +This produces a sample $\mathbf{x} \sim \mathcal{N}_k(\boldsymbol{\mu}, \boldsymbol{\Sigma})$. The Cholesky approach is preferred because it is computationally efficient ($O(k^3)$ once, then $O(k^2)$ per sample), and the decomposition itself serves as a check that $\boldsymbol{\Sigma}$ is positive definite. + ### Creating Multivariate Normal Distributions ```cs @@ -74,16 +133,7 @@ Console.WriteLine($" f(x) = {pdf:E4}"); Console.WriteLine($" log f(x) = {logPdf:F4}"); ``` -**Formula:** -``` -f(x) = (2π)^(-k/2) |Σ|^(-1/2) exp[-½(x-μ)ᵀΣ⁻¹(x-μ)] - -Where: -- k = dimension -- μ = mean vector -- Σ = covariance matrix -- |Σ| = determinant of Σ -``` +The PDF formula is given in the [Mathematical Definition](#mathematical-definition) section above. The library computes it as $f(\mathbf{x}) = \exp(-\tfrac{1}{2}D^2 + \ln C)$, where $D^2$ is the Mahalanobis distance and $\ln C = -\tfrac{1}{2}(k \ln 2\pi + \ln|\boldsymbol{\Sigma}|)$ is a precomputed constant. ### Cumulative Distribution Function (CDF) @@ -124,21 +174,27 @@ Console.WriteLine($" P(Z₁ ≤ {z1}, Z₂ ≤ {z2} | ρ={rho}) = {bivCDF:F6}") ### Inverse CDF (Quantile Function) +The `InverseCDF(double[] probabilities)` method takes one probability per dimension and returns a single point in k-dimensional space where each marginal is at its respective probability level: + ```cs -// Generate quantiles for each marginal -double[] probabilities = { 0.05, 0.50, 0.95 }; +// Get the point where dim1 is at 5%, dim2 is at 50%, dim3 is at 95% +double[] point = mvn.InverseCDF(new double[] { 0.05, 0.50, 0.95 }); -double[] quantiles = mvn.InverseCDF(probabilities); +Console.WriteLine("Multivariate quantile point:"); +for (int i = 0; i < mvn.Dimension; i++) +{ + Console.WriteLine($" X{i + 1} at p={new[] { 0.05, 0.50, 0.95 }[i]}: {point[i]:F2}"); +} -Console.WriteLine("Marginal quantiles:"); +// For marginal quantiles, use the marginal normal distributions directly +Console.WriteLine("\nMarginal quantile ranges:"); for (int i = 0; i < mvn.Dimension; i++) { - double q05 = mvn.Mean[i] + Math.Sqrt(mvn.Covariance[i, i]) * - new Normal(0, 1).InverseCDF(0.05); + double sd = Math.Sqrt(mvn.Covariance[i, i]); + double q05 = mvn.Mean[i] + sd * new Normal(0, 1).InverseCDF(0.05); double q50 = mvn.Mean[i]; - double q95 = mvn.Mean[i] + Math.Sqrt(mvn.Covariance[i, i]) * - new Normal(0, 1).InverseCDF(0.95); - + double q95 = mvn.Mean[i] + sd * new Normal(0, 1).InverseCDF(0.95); + Console.WriteLine($" X{i + 1}: 5%={q05:F2}, 50%={q50:F2}, 95%={q95:F2}"); } ``` @@ -280,8 +336,8 @@ for (int i = 0; i < nYears; i++) Console.WriteLine($"Portfolio Analysis ({nYears} simulations):"); Console.WriteLine($" Mean return: {portfolioReturns.Average():P2}"); Console.WriteLine($" Volatility: {Statistics.StandardDeviation(portfolioReturns):P2}"); -Console.WriteLine($" 5th percentile: {Statistics.Percentile(portfolioReturns.OrderBy(r => r).ToArray(), 5):P2}"); -Console.WriteLine($" 95th percentile: {Statistics.Percentile(portfolioReturns.OrderBy(r => r).ToArray(), 95):P2}"); +Console.WriteLine($" 5th percentile: {Statistics.Percentile(portfolioReturns, 0.05):P2}"); +Console.WriteLine($" 95th percentile: {Statistics.Percentile(portfolioReturns, 0.95):P2}"); ``` ### Example 2: Multivariate Exceedance Probability @@ -388,7 +444,7 @@ var diff = new Vector(x.Select((xi, i) => xi - means[i]).ToArray()); var covMatrix = new Matrix(cov); var covInv = covMatrix.Inverse(); -double mahalanobis = Math.Sqrt(diff.DotProduct(covInv.Multiply(diff))); +double mahalanobis = Math.Sqrt(Vector.DotProduct(diff, covInv.Multiply(diff))); // Chi-squared test: D² ~ χ²(k) under null hypothesis double chiSqCritical = 7.815; // 95th percentile of χ²(3) @@ -405,6 +461,269 @@ else Console.WriteLine(" → Point is not an outlier"); ``` +## Multivariate Student-t Distribution + +The multivariate Student-t distribution generalizes the univariate Student-t to multiple dimensions, providing heavier tails than the multivariate normal. It is used for robust modeling when data may contain outliers [[2]](#2). + +### Mathematical Definition + +A random vector $\mathbf{X} = (X_1, \ldots, X_k)^T$ follows a $k$-dimensional multivariate Student-t distribution, written $\mathbf{X} \sim t_k(\nu, \boldsymbol{\mu}, \boldsymbol{\Sigma})$, if its probability density function is: + +```math +f(\mathbf{x}) = \frac{\Gamma\!\left(\frac{\nu + k}{2}\right)}{\Gamma\!\left(\frac{\nu}{2}\right) (\nu\pi)^{k/2} |\boldsymbol{\Sigma}|^{1/2}} \left[ 1 + \frac{1}{\nu} (\mathbf{x} - \boldsymbol{\mu})^T \boldsymbol{\Sigma}^{-1} (\mathbf{x} - \boldsymbol{\mu}) \right]^{-(\nu+k)/2} +``` + +where: +- $\nu > 0$ is the degrees of freedom, +- $\boldsymbol{\mu} \in \mathbb{R}^k$ is the location vector, +- $\boldsymbol{\Sigma}$ is the $k \times k$ positive-definite scale matrix (not the covariance matrix). + +**Important:** The scale matrix $\boldsymbol{\Sigma}$ is distinct from the covariance matrix. The moments are: + +```math +\text{Mean} = \boldsymbol{\mu} \quad (\text{for } \nu > 1), \qquad \text{Cov}(\mathbf{X}) = \frac{\nu}{\nu - 2}\,\boldsymbol{\Sigma} \quad (\text{for } \nu > 2) +``` + +The mean is undefined when $\nu \leq 1$, and the covariance is undefined when $\nu \leq 2$. + +**Relationship to MVN.** As $\nu \to \infty$, the multivariate Student-t distribution converges to the multivariate normal $\mathcal{N}_k(\boldsymbol{\mu}, \boldsymbol{\Sigma})$. For finite $\nu$, the tails are heavier, controlled by the degrees of freedom parameter. This makes the multivariate Student-t distribution useful for robust statistical inference where normal approximations may underestimate tail probabilities. + +**Sampling algorithm.** The multivariate Student-t can be represented as a scale mixture of normals. A sample is generated as: + +```math +\mathbf{x} = \boldsymbol{\mu} + \frac{\mathbf{L}\mathbf{z}}{\sqrt{W/\nu}} +``` + +where $\mathbf{L}$ is the Cholesky factor of $\boldsymbol{\Sigma}$, $\mathbf{z} \sim \mathcal{N}(\mathbf{0}, \mathbf{I}_k)$, and $W \sim \chi^2(\nu)$. The library generates $W$ via $W \sim \text{Gamma}(\nu/2, 2)$ to support non-integer degrees of freedom. + +### Creating Multivariate Student-t Distributions + +```cs +using Numerics.Distributions; + +// Bivariate Student-t with 5 degrees of freedom +double nu = 5; +double[] mu = { 100, 50 }; +double[,] sigma = { + { 225, 75 }, + { 75, 100 } +}; + +var mvt = new MultivariateStudentT(nu, mu, sigma); + +Console.WriteLine($"Multivariate Student-t Distribution:"); +Console.WriteLine($" Dimension: {mvt.Dimension}"); +Console.WriteLine($" Degrees of freedom: {nu}"); +``` + +### PDF, CDF, and Sampling + +```cs +// Evaluate density +double[] x = { 105, 55 }; +double pdf = mvt.PDF(x); +double logPdf = mvt.LogPDF(x); + +Console.WriteLine($"PDF at x: {pdf:E4}"); +Console.WriteLine($"Log-PDF at x: {logPdf:F4}"); + +// CDF +double cdf = mvt.CDF(x); +Console.WriteLine($"CDF at x: {cdf:F6}"); + +// Generate random samples +double[,] samples = mvt.GenerateRandomValues(1000, seed: 42); +``` + +## Dirichlet Distribution + +The Dirichlet distribution $\text{Dir}(\alpha_1, \alpha_2, \ldots, \alpha_k)$ is a multivariate generalization of the Beta distribution, defined on the probability simplex (components sum to 1). It is the conjugate prior for the categorical and multinomial distributions in Bayesian statistics [[3]](#3). + +### Mathematical Definition + +A random vector $\mathbf{X} = (X_1, X_2, \ldots, X_k)$ follows a Dirichlet distribution $\text{Dir}(\alpha_1, \ldots, \alpha_k)$ if its probability density function on the $(k-1)$-dimensional simplex is: + +```math +f(x_1, \ldots, x_k) = \frac{\Gamma\!\left(\sum_{i=1}^{k} \alpha_i\right)}{\prod_{i=1}^{k} \Gamma(\alpha_i)} \prod_{i=1}^{k} x_i^{\alpha_i - 1} +``` + +with constraints $x_i > 0$ for all $i$ and $\sum_{i=1}^{k} x_i = 1$. Each $\alpha_i > 0$ is a concentration parameter. The normalizing constant involves the multivariate Beta function $B(\boldsymbol{\alpha}) = \prod \Gamma(\alpha_i) / \Gamma(\sum \alpha_i)$. + +**Moments.** Let $\alpha_0 = \sum_{i=1}^{k} \alpha_i$ denote the total concentration. Then: + +```math +\text{E}[X_i] = \frac{\alpha_i}{\alpha_0}, \qquad \text{Var}(X_i) = \frac{\alpha_i (\alpha_0 - \alpha_i)}{\alpha_0^2 (\alpha_0 + 1)} +``` + +```math +\text{Cov}(X_i, X_j) = \frac{-\alpha_i \alpha_j}{\alpha_0^2 (\alpha_0 + 1)} \quad (i \neq j) +``` + +The negative covariance reflects that components are constrained to sum to 1 -- if one component increases, others must decrease. The mode exists when all $\alpha_i > 1$ and is given by $\text{Mode}(X_i) = (\alpha_i - 1) / (\alpha_0 - k)$. + +**Connection to Beta distribution.** For $k = 2$, the Dirichlet reduces to the Beta distribution: if $(X_1, X_2) \sim \text{Dir}(\alpha_1, \alpha_2)$, then $X_1 \sim \text{Beta}(\alpha_1, \alpha_2)$. + +**Sampling algorithm.** Samples are generated using the gamma distribution representation. Draw $k$ independent gamma variates $Y_i \sim \text{Gamma}(\alpha_i, 1)$, then normalize: + +```math +X_i = \frac{Y_i}{\sum_{j=1}^{k} Y_j} +``` + +The resulting vector $(X_1, \ldots, X_k)$ lies on the simplex and follows the Dirichlet distribution. + +**Use cases.** The Dirichlet distribution arises naturally when modeling proportions or mixture weights: Bayesian priors for categorical data, topic modeling (document-topic proportions), compositional data analysis, and mixture model weight estimation in RMC-BestFit. + +### Creating Dirichlet Distributions + +```cs +using Numerics.Distributions; + +// Symmetric Dirichlet with K=3 categories, all alpha=2 +var dir1 = new Dirichlet(dimension: 3, alpha: 2.0); + +// Asymmetric Dirichlet with specified concentration parameters +var dir2 = new Dirichlet(new double[] { 1.0, 2.0, 5.0 }); + +Console.WriteLine($"Dirichlet Distribution:"); +Console.WriteLine($" Dimension: {dir2.Dimension}"); +Console.WriteLine($" Alpha sum: {dir2.AlphaSum}"); +``` + +### Properties + +```cs +var dir = new Dirichlet(new double[] { 2.0, 3.0, 5.0 }); + +// Mean vector: E[Xi] = alpha_i / sum(alpha) +double[] mean = dir.Mean; +Console.WriteLine($"Mean: [{string.Join(", ", mean.Select(m => m.ToString("F3")))}]"); + +// Variance vector +double[] variance = dir.Variance; +Console.WriteLine($"Variance: [{string.Join(", ", variance.Select(v => v.ToString("F4")))}]"); + +// Mode (requires all alpha_i > 1) +double[] mode = dir.Mode; +Console.WriteLine($"Mode: [{string.Join(", ", mode.Select(m => m.ToString("F3")))}]"); + +// Covariance between components +double cov01 = dir.Covariance(0, 1); // Negative (components are negatively correlated) +Console.WriteLine($"Cov(X0, X1): {cov01:F4}"); +``` + +### PDF and Sampling + +```cs +// Evaluate density at a point on the simplex +double[] x = { 0.2, 0.3, 0.5 }; +double pdf = dir.PDF(x); +double logPdf = dir.LogPDF(x); + +Console.WriteLine($"PDF at x: {pdf:F6}"); +Console.WriteLine($"Log-PDF at x: {logPdf:F4}"); + +// Generate random samples (each row sums to 1) +double[,] samples = dir.GenerateRandomValues(1000, seed: 42); +``` + +### Application: Bayesian Mixture Weights + +```cs +// Prior for 3-component mixture model weights +var prior = new Dirichlet(dimension: 3, alpha: 1.0); // Uniform on simplex + +// After observing data, update to posterior +// (conjugate update: add counts to alpha) +int[] counts = { 50, 120, 80 }; +double[] posteriorAlpha = { 1.0 + counts[0], 1.0 + counts[1], 1.0 + counts[2] }; +var posterior = new Dirichlet(posteriorAlpha); + +Console.WriteLine($"Posterior mean weights: [{string.Join(", ", posterior.Mean.Select(m => m.ToString("F3")))}]"); +``` + +## Multinomial Distribution + +The Multinomial distribution models the number of outcomes in each of $k$ categories over $n$ independent trials, where each trial has a fixed probability vector. It generalizes the Binomial distribution to more than two categories [[4]](#4). + +### Mathematical Definition + +A random vector $\mathbf{X} = (X_1, X_2, \ldots, X_k)$ follows a multinomial distribution $\text{Mult}(n, \mathbf{p})$ with $n$ trials and probability vector $\mathbf{p} = (p_1, \ldots, p_k)$ if its probability mass function is: + +```math +P(X_1 = x_1, \ldots, X_k = x_k) = \frac{n!}{\prod_{i=1}^{k} x_i!} \prod_{i=1}^{k} p_i^{x_i} +``` + +with constraints $x_i \in \lbrace 0, 1, \ldots, n\rbrace$, $\sum_{i=1}^{k} x_i = n$, $p_i \geq 0$, and $\sum_{i=1}^{k} p_i = 1$. + +The multinomial coefficient $n! / \prod x_i!$ counts the number of ways to arrange $n$ trials into $k$ categories with the specified counts. + +**Moments:** + +```math +\text{E}[X_i] = n p_i, \qquad \text{Var}(X_i) = n p_i (1 - p_i) +``` + +```math +\text{Cov}(X_i, X_j) = -n p_i p_j \quad (i \neq j) +``` + +The negative covariance follows from the constraint $\sum X_i = n$: observing more counts in one category necessarily reduces the counts available for other categories. + +**Connection to Binomial.** For $k = 2$, the multinomial reduces to the binomial distribution: if $(X_1, X_2) \sim \text{Mult}(n, (p, 1-p))$, then $X_1 \sim \text{Binomial}(n, p)$. + +**Sampling algorithm.** The library generates samples via sequential conditional binomial draws. For each category $i = 1, \ldots, k-1$, draw $X_i \sim \text{Binomial}(n_{\text{remaining}}, p_i / p_{\text{remaining}})$, then set the last category to the remainder. This method is exact and efficient. + +**Use cases.** The multinomial distribution is used for modeling count data across categories: MCMC trajectory state selection in the NUTS sampler, categorical data modeling in LifeSim, and as the likelihood function for categorical observations in Bayesian inference. + +### Creating Multinomial Distributions + +```cs +using Numerics.Distributions; + +// 10 trials with 3 categories having probabilities 0.2, 0.3, 0.5 +var mult = new Multinomial(10, new double[] { 0.2, 0.3, 0.5 }); + +Console.WriteLine($"Multinomial Distribution:"); +Console.WriteLine($" Dimension: {mult.Dimension}"); +Console.WriteLine($" Number of trials: {mult.NumberOfTrials}"); +``` + +### Properties + +```cs +// Mean vector: E[Xi] = N * p_i +double[] mean = mult.Mean; +Console.WriteLine($"Mean: [{string.Join(", ", mean.Select(m => m.ToString("F1")))}]"); + +// Variance vector: Var[Xi] = N * p_i * (1 - p_i) +double[] variance = mult.Variance; +Console.WriteLine($"Variance: [{string.Join(", ", variance.Select(v => v.ToString("F2")))}]"); + +// Covariance: Cov(Xi, Xj) = -N * p_i * p_j +double cov01 = mult.Covariance(0, 1); +Console.WriteLine($"Cov(X0, X1): {cov01:F2}"); +``` + +### PMF and Sampling + +```cs +// Probability mass function +double[] counts = { 2, 3, 5 }; +double pmf = mult.PDF(counts); +double logPmf = mult.LogPMF(counts); + +Console.WriteLine($"P(X = [2,3,5]): {pmf:F6}"); + +// Generate random samples (each row sums to N) +double[,] samples = mult.GenerateRandomValues(1000, seed: 42); + +// Weighted categorical sampling (static utility) +var rng = new Random(42); +double[] weights = { 1.0, 3.0, 6.0 }; +int category = Multinomial.Sample(weights, rng); +Console.WriteLine($"Sampled category: {category}"); +``` + ## Key Concepts ### Positive Definiteness @@ -475,21 +794,157 @@ Console.WriteLine("Independent MVN → Diagonal covariance matrix"); 6. **Conditional distributions** - Use formula for bivariate case 7. **Outlier detection** - Use Mahalanobis distance +## Sampling Algorithms + +This section summarizes the algorithms used by the library to generate random samples from each multivariate distribution. + +### MVN via Cholesky Decomposition + +The `MultivariateNormal.GenerateRandomValues` method uses the Cholesky decomposition $\boldsymbol{\Sigma} = \mathbf{L}\mathbf{L}^T$ to transform independent standard normal variates into correlated samples: + +```math +\mathbf{z} \sim \mathcal{N}(\mathbf{0}, \mathbf{I}_k), \quad \mathbf{x} = \boldsymbol{\mu} + \mathbf{L}\mathbf{z} +``` + +The Cholesky approach is preferred for three reasons: (1) it is computationally efficient -- the decomposition is $O(k^3)$ but computed only once, after which each sample requires only $O(k^2)$; (2) the decomposition succeeds if and only if $\boldsymbol{\Sigma}$ is positive definite, providing a built-in validity check; and (3) the triangular structure of $\mathbf{L}$ makes the matrix-vector product efficient. The library also provides `LatinHypercubeRandomValues` for improved space-filling properties via stratified sampling. + +### Multivariate Student-t Sampling + +The `MultivariateStudentT.GenerateRandomValues` method exploits the scale-mixture representation. A multivariate Student-t variate can be constructed by scaling a multivariate normal variate by an independent chi-squared random variable: + +```math +\mathbf{z} \sim \mathcal{N}(\mathbf{0}, \mathbf{I}_k), \quad W \sim \chi^2(\nu), \quad \mathbf{x} = \boldsymbol{\mu} + \mathbf{L}\mathbf{z} \cdot \sqrt{\frac{\nu}{W}} +``` + +The library generates $W$ via $W \sim \text{Gamma}(\nu/2, 2)$ to support non-integer degrees of freedom. + +### Dirichlet Sampling + +The `Dirichlet.GenerateRandomValues` method uses the gamma distribution representation. Draw $k$ independent gamma variates and normalize: + +```math +Y_i \sim \text{Gamma}(\alpha_i, 1), \quad X_i = \frac{Y_i}{\sum_{j=1}^{k} Y_j} +``` + +### Multinomial Sampling + +The `Multinomial.GenerateRandomValues` method uses sequential conditional binomial draws. For category $i = 1, \ldots, k-1$: + +```math +X_i \sim \text{Binomial}\!\left(n_{\text{remaining}},\; \frac{p_i}{p_{\text{remaining}}}\right), \quad X_k = n - \sum_{i=1}^{k-1} X_i +``` + +This method is exact and avoids the need to enumerate the full multinomial support. The library uses direct simulation for small $n$ and the BTPE algorithm for large $n$. + ## Computational Notes -- **PDF**: O(k³) for Cholesky decomposition -- **CDF**: O(k) for k≤2, O(n·k) for k>2 (Monte Carlo with n evaluations) -- **Sampling**: O(k²) per sample using Cholesky -- **Storage**: O(k²) for covariance matrix +- **PDF**: O(k^3) for Cholesky decomposition (one-time), O(k^2) per evaluation +- **CDF**: O(k) for k<=2, O(n*k) for k>2 (Monte Carlo with n evaluations) +- **Sampling**: O(k^2) per sample using Cholesky (MVN and Student-t) +- **Storage**: O(k^2) for covariance/scale matrix + +--- + +## Bivariate Empirical Distribution + +The `BivariateEmpirical` class represents a bivariate empirical CDF defined on a grid of values. It is useful when the joint distribution is known only through tabulated CDF values rather than a parametric form. + +```cs +using Numerics.Distributions; +using Numerics.Data; + +// Define grid values +double[] x1Values = { 100, 200, 300, 400, 500 }; // e.g., flow (cfs) +double[] x2Values = { 10, 20, 30, 40 }; // e.g., stage (ft) + +// Define CDF values on the grid (must be non-decreasing, values in [0, 1]) +double[,] pValues = { + { 0.01, 0.02, 0.03, 0.04 }, + { 0.05, 0.15, 0.20, 0.22 }, + { 0.10, 0.35, 0.50, 0.55 }, + { 0.15, 0.50, 0.75, 0.82 }, + { 0.20, 0.60, 0.90, 1.00 } +}; + +// Create bivariate empirical distribution +var bivariate = new BivariateEmpirical(x1Values, x2Values, pValues); + +// Evaluate CDF at a point +double prob = bivariate.CDF(250, 25); +Console.WriteLine($"P(X1 ≤ 250, X2 ≤ 25) = {prob:F4}"); + +// Optional: apply transforms for interpolation +var bivariateLog = new BivariateEmpirical( + x1Values, x2Values, pValues, + x1Transform: Transform.Logarithmic, + x2Transform: Transform.None, + probabilityTransform: Transform.NormalZ +); +``` + +**Properties:** +- `X1Values`, `X2Values` — grid axis values +- `ProbabilityValues` — CDF values on the grid +- `Dimension` — always 2 +- Supports `Transform.None`, `Transform.Logarithmic`, and `Transform.NormalZ` for interpolation + +**Note:** The `PDF()` method returns `double.NaN` because this is a purely empirical CDF — no density function is defined. + +### Practical Example: Joint Flood Stage-Duration Exceedance + +```cs +using Numerics.Distributions; +using Numerics.Data; + +// Define flood stage grid (ft) +double[] stages = { 10, 12, 14, 16, 18, 20 }; +// Define flood duration grid (hr) +double[] durations = { 6, 12, 24, 48, 72 }; + +// Joint CDF values P(Stage ≤ s, Duration ≤ d) +// Based on observed flood event records +double[,] jointCDF = { + { 0.30, 0.35, 0.38, 0.39, 0.40 }, + { 0.40, 0.50, 0.55, 0.57, 0.58 }, + { 0.45, 0.60, 0.70, 0.73, 0.75 }, + { 0.48, 0.65, 0.78, 0.83, 0.85 }, + { 0.49, 0.68, 0.82, 0.90, 0.93 }, + { 0.50, 0.70, 0.85, 0.95, 1.00 } +}; + +// Create with log transform on stages for better interpolation +var jointDist = new BivariateEmpirical( + stages, durations, jointCDF, + x1Transform: Transform.Logarithmic, + x2Transform: Transform.None, + probabilityTransform: Transform.NormalZ +); + +// Probability that a flood has stage ≤ 15 ft AND duration ≤ 36 hr +double prob = jointDist.CDF(15, 36); +Console.WriteLine($"P(Stage ≤ 15, Duration ≤ 36hr) = {prob:F4}"); + +// Joint exceedance probability +double jointExceedance = 1 - jointDist.CDF(16, 48); +Console.WriteLine($"P(Stage > 16 AND/OR Duration > 48hr) = {jointExceedance:F4}"); +``` --- ## References -[1] Tong, Y. L. (2012). *The Multivariate Normal Distribution*. Springer Science & Business Media. +[1] Anderson, T. W. (2003). *An Introduction to Multivariate Statistical Analysis* (3rd ed.). Wiley. + +[2] Kotz, S., Balakrishnan, N., & Johnson, N. L. (2000). *Continuous Multivariate Distributions, Volume 1: Models and Applications* (2nd ed.). Wiley. + +[3] Kotz, S., Balakrishnan, N. & Johnson, N. L. (2000). *Continuous Multivariate Distributions, Volume 1: Models and Applications* (2nd ed.). Wiley. Chapter 49 (Dirichlet distribution). + +[4] Johnson, N. L., Kotz, S. & Balakrishnan, N. (1997). *Discrete Multivariate Distributions*. Wiley. Chapter 35 (Multinomial distribution). + +[5] Tong, Y. L. (2012). *The Multivariate Normal Distribution*. Springer Science & Business Media. -[2] Kotz, S., Balakrishnan, N., & Johnson, N. L. (2004). *Continuous Multivariate Distributions, Volume 1: Models and Applications* (2nd ed.). John Wiley & Sons. +[6] Kotz, S. & Nadarajah, S. (2004). *Multivariate t Distributions and Their Applications*. Cambridge University Press. --- -[← Previous: Copulas](copulas.md) | [Back to Index](../index.md) +[← Previous: Copulas](copulas.md) | [Back to Index](../index.md) | [Next: Machine Learning →](../machine-learning/machine-learning.md) diff --git a/docs/distributions/parameter-estimation.md b/docs/distributions/parameter-estimation.md index 50141250..62e5b76e 100644 --- a/docs/distributions/parameter-estimation.md +++ b/docs/distributions/parameter-estimation.md @@ -27,7 +27,7 @@ public enum ParameterEstimationMethod | **Method of Moments** | Simple, fast | Inefficient, biased for small samples | Quick estimates, stable parameters | | **Method of Percentiles** | Intuitive, robust | Less efficient | Expert judgment, special cases | -**Recommendation for Hydrological Applications:** L-Moments are recommended by USGS [[1]](#1) for flood frequency analysis due to superior performance with small samples and robustness to outliers. +**Recommendation for Hydrological Applications:** L-moments are recommended by USGS [[1]](#1) for flood frequency analysis due to superior performance with small samples and robustness to outliers. ## Using the Estimate() Method @@ -78,7 +78,7 @@ double[] data = { 12500, 15300, 11200, 18700, 14100, 16800, 13400, 17200 }; // Step 1: Compute L-moments from data double[] lMoments = Statistics.LinearMoments(data); -Console.WriteLine("Sample L-Moments:"); +Console.WriteLine("Sample L-moments:"); Console.WriteLine($" λ₁ (mean): {lMoments[0]:F2}"); Console.WriteLine($" λ₂ (L-scale): {lMoments[1]:F2}"); Console.WriteLine($" τ₃ (L-skewness): {lMoments[2]:F4}"); @@ -146,12 +146,58 @@ L-moments are linear combinations of order statistics that provide robust altern - Hydrological applications - Extreme value analysis +### Mathematical Formulation + +L-moments are defined through probability-weighted moments (PWMs). For a random variable $X$ with CDF $F(x)$, the probability-weighted moments are: + +```math +\beta_r = E\left[X \cdot F(X)^r\right] = \int_0^1 x(F) \cdot F^r \, dF, \quad r = 0, 1, 2, \ldots +``` + +The first four L-moments are linear combinations of the PWMs: + +```math +\lambda_1 = \beta_0 +``` + +```math +\lambda_2 = 2\beta_1 - \beta_0 +``` + +```math +\lambda_3 = 6\beta_2 - 6\beta_1 + \beta_0 +``` + +```math +\lambda_4 = 20\beta_3 - 30\beta_2 + 12\beta_1 - \beta_0 +``` + +The L-moment ratios, which are dimensionless and bounded, are defined as: + +```math +\tau = \frac{\lambda_2}{\lambda_1} \quad \text{(L-CV)}, \qquad \tau_3 = \frac{\lambda_3}{\lambda_2} \quad \text{(L-skewness)}, \qquad \tau_4 = \frac{\lambda_4}{\lambda_2} \quad \text{(L-kurtosis)} +``` + +L-skewness is bounded in $[-1, 1]$ and L-kurtosis in $[\frac{1}{4}(5\tau_3^2 - 1),\; 1]$, unlike conventional skewness and kurtosis which are unbounded. This boundedness makes L-moment ratios more interpretable and stable. + +**Sample estimation.** Given a sorted sample $x_{1:n} \leq x_{2:n} \leq \cdots \leq x_{n:n}$, the unbiased sample PWM estimators are: + +```math +b_r = \frac{1}{n}\sum_{j=r+1}^{n} \frac{\binom{j-1}{r}}{\binom{n-1}{r}} \, x_{j:n}, \quad r = 0, 1, 2, \ldots +``` + +The `Statistics.LinearMoments()` method computes these sample PWMs and returns the array $[\lambda_1, \lambda_2, \tau_3, \tau_4]$. + +**Why L-moments are preferred for small samples.** Conventional moments involve powers of deviations from the mean, so a single extreme observation can dominate the skewness or kurtosis estimate. L-moments use only linear combinations of order statistics, which makes them far more robust to outliers and nearly unbiased even for samples as small as $n = 10$. For hydrological applications where sample sizes are often 30--60 years of annual data, this robustness is critical. + +**L-moment ratio diagrams.** Plotting sample L-skewness ($\tau_3$) against L-kurtosis ($\tau_4$) and comparing to the theoretical curves of candidate distributions is a powerful tool for distribution identification. Each distribution family traces a distinct curve (or point) in L-moment ratio space, making visual comparison straightforward [[2]](#2). + ### Properties of L-Moments -1. **More robust** than conventional moments - less influenced by outliers +1. **More robust** than conventional moments -- less influenced by outliers 2. **Less biased** for small samples -3. **More efficient** - smaller sampling variance -4. **Bounded** - L-moment ratios are bounded, unlike conventional moments +3. **More efficient** -- smaller sampling variance +4. **Bounded** -- L-moment ratios are bounded, unlike conventional moments 5. **Nearly unbiased** even for very small samples (n = 10) ### Computing L-Moments @@ -213,7 +259,53 @@ Console.WriteLine($"{"Sample",-12} | {sampleLM[2],11:F4} | {sampleLM[3],11:F4}") ## Maximum Likelihood Estimation -MLE finds parameters that maximize the likelihood of observing the data [[3]](#3): +Maximum Likelihood Estimation (MLE) finds the parameter values that make the observed data most probable under the assumed model [[3]](#3). + +### Mathematical Formulation + +Given independent observations $x_1, x_2, \ldots, x_n$ from a distribution with PDF $f(x|\boldsymbol{\theta})$, the likelihood function is the joint probability of the data viewed as a function of the parameters: + +```math +L(\boldsymbol{\theta} \,|\, \mathbf{x}) = \prod_{i=1}^{n} f(x_i \,|\, \boldsymbol{\theta}) +``` + +Because products are numerically unstable, optimization is performed on the log-likelihood: + +```math +\ell(\boldsymbol{\theta}) = \sum_{i=1}^{n} \log f(x_i \,|\, \boldsymbol{\theta}) +``` + +The MLE is the parameter vector that maximizes the log-likelihood: + +```math +\hat{\boldsymbol{\theta}}_{\text{MLE}} = \underset{\boldsymbol{\theta}}{\text{argmax}} \; \ell(\boldsymbol{\theta}) +``` + +For some distributions (e.g., Normal, Exponential), the MLE has a closed-form solution. For most distributions used in hydrology (GEV, LP3, Weibull), the optimization must be solved numerically. The library uses constrained optimization with initial values derived from L-moment estimates. + +**Fisher Information and standard errors.** The Fisher Information matrix quantifies the curvature of the log-likelihood surface at the maximum: + +```math +\mathcal{I}(\boldsymbol{\theta}) = -E\left[\frac{\partial^2 \ell}{\partial \boldsymbol{\theta} \, \partial \boldsymbol{\theta}^T}\right] +``` + +Under regularity conditions, the MLE is asymptotically normal [[5]](#5): + +```math +\sqrt{n}\left(\hat{\boldsymbol{\theta}} - \boldsymbol{\theta}\right) \xrightarrow{d} N\left(\mathbf{0},\; \mathcal{I}(\boldsymbol{\theta})^{-1}\right) \quad \text{as } n \to \infty +``` + +This provides approximate standard errors for each parameter: + +```math +\text{SE}(\hat{\theta}_j) \approx \frac{1}{\sqrt{\mathcal{I}(\hat{\boldsymbol{\theta}})_{jj}}} +``` + +**Strengths:** Asymptotically efficient (achieves the lowest possible variance among consistent estimators), asymptotically unbiased, invariant under reparameterization, provides a natural framework for model comparison via AIC and BIC. + +**Weaknesses:** Requires numerical optimization that may fail to converge, sensitive to outliers, can be biased and inefficient for small samples, requires specification of the full probability model. + +### Using MLE ```cs using Numerics.Distributions; @@ -225,8 +317,8 @@ var weibull = new Weibull(); weibull.Estimate(observations, ParameterEstimationMethod.MaximumLikelihood); Console.WriteLine($"Weibull Parameters (MLE):"); -Console.WriteLine($" Scale (α): {weibull.Alpha:F3}"); -Console.WriteLine($" Shape (β): {weibull.Beta:F3}"); +Console.WriteLine($" Scale (λ): {weibull.Lambda:F3}"); +Console.WriteLine($" Shape (κ): {weibull.Kappa:F3}"); // Compute log-likelihood at fitted parameters double logLikelihood = 0; @@ -275,7 +367,49 @@ catch (Exception ex) ## Method of Moments -MOM matches sample moments with theoretical moments: +The Method of Moments (MOM) is the oldest and simplest approach to parameter estimation. The core idea is to equate sample moments to the corresponding theoretical moments of the distribution and solve for the unknown parameters. + +### Mathematical Formulation + +Given a sample $x_1, x_2, \ldots, x_n$, the first four sample moments are the mean, standard deviation, skewness, and kurtosis: + +```math +\bar{x} = \frac{1}{n}\sum_{i=1}^{n} x_i +``` + +```math +s = \sqrt{\frac{1}{n-1}\sum_{i=1}^{n}(x_i - \bar{x})^2} +``` + +```math +\hat{\gamma} = \frac{n}{(n-1)(n-2)} \sum_{i=1}^{n}\left(\frac{x_i - \bar{x}}{s}\right)^3 +``` + +```math +\hat{\kappa} = \frac{n(n+1)}{(n-1)(n-2)(n-3)} \sum_{i=1}^{n}\left(\frac{x_i - \bar{x}}{s}\right)^4 - \frac{3(n-1)^2}{(n-2)(n-3)} +``` + +The `Statistics.ProductMoments()` method returns these four quantities as the array $[\bar{x}, s, \hat{\gamma}, \hat{\kappa}]$. + +MOM estimation sets the theoretical moments equal to the sample moments and solves for the distribution parameters. For a two-parameter distribution, only the first two moments (mean and standard deviation) are needed. For three-parameter distributions, skewness is also required. + +**Example: Normal distribution.** The Normal($\mu$, $\sigma$) has $E[X] = \mu$ and $\text{SD}[X] = \sigma$. Equating sample to theoretical moments yields: + +```math +\hat{\mu} = \bar{x}, \quad \hat{\sigma} = s +``` + +**Example: Gamma distribution.** The Gamma($\kappa$, $\theta$) has $E[X] = \kappa\theta$ and $\text{Var}[X] = \kappa\theta^2$. Solving for the parameters: + +```math +\hat{\kappa} = \frac{\bar{x}^2}{s^2}, \quad \hat{\theta} = \frac{s^2}{\bar{x}} +``` + +**Strengths:** Simple, closed-form solutions, always produces estimates, computationally fast. + +**Weaknesses:** Not statistically efficient (higher variance than MLE), can produce invalid parameters for skewed distributions, estimates are sensitive to outliers because conventional moments give disproportionate weight to extreme values. + +### Using Method of Moments ```cs double[] data = { 100, 105, 98, 110, 95, 102, 108, 97, 103, 106 }; @@ -298,6 +432,53 @@ Console.WriteLine($" Sample mean = {moments[0]:F2}"); Console.WriteLine($" Sample std dev = {moments[1]:F2}"); ``` +## Method of Percentiles + +The Method of Percentiles (also called least-squares fitting or quantile matching) estimates parameters by matching theoretical quantiles of the distribution to empirical quantiles computed from the data. + +### Mathematical Formulation + +Given a sorted sample $x_{(1)} \leq x_{(2)} \leq \cdots \leq x_{(n)}$, each observation is assigned a plotting position $p_i$ that estimates $F(x_{(i)})$. A common choice is the Weibull plotting position: + +```math +p_i = \frac{i}{n + 1} +``` + +The parameters $\boldsymbol{\theta}$ are then chosen so that the theoretical quantile function (inverse CDF) matches the observed data as closely as possible. For a distribution with quantile function $F^{-1}(p;\,\boldsymbol{\theta})$, the parameters minimize the sum of squared differences: + +```math +\hat{\boldsymbol{\theta}} = \underset{\boldsymbol{\theta}}{\text{argmin}} \sum_{i=1}^{n} \left[x_{(i)} - F^{-1}(p_i;\,\boldsymbol{\theta})\right]^2 +``` + +For a two-parameter distribution, it is sufficient to select two percentiles (e.g., the median and the 84th percentile) and solve the resulting system of two equations: + +```math +F^{-1}(p_j;\,\boldsymbol{\theta}) = x_{(j)}, \quad j \in \{j_1,\, j_2\} +``` + +**Strengths:** Intuitive and easy to visualize, always produces estimates, moderately robust to outliers in the tails, useful when expert judgment suggests specific quantile targets. + +**Weaknesses:** Uses only selected data points or gives equal weight to all quantiles (not statistically efficient), lower precision than MLE or L-moments for most distributions. + +## Estimation Method Comparison + +The choice of estimation method depends on sample size, data quality, and application requirements. The following table summarizes the key trade-offs: + +| Method | Efficiency | Small Samples | Robustness | Complexity | Best For | +|--------|-----------|---------------|-----------|-----------|---------| +| **MOM** | Low | Fair | Low | Simple | Quick estimates, stable distributions | +| **L-Moments** | Moderate--High | Excellent | High | Moderate | Hydrological data, small samples | +| **MLE** | Highest (asymptotic) | Poor--Fair | Low | Complex | Large samples, model comparison | +| **Percentiles** | Low | Fair | Moderate | Simple | Visual fitting, expert judgment | + +### Rules of Thumb + +- **n < 50:** Prefer L-moments. With small samples, robustness matters more than asymptotic efficiency, and L-moment estimates are nearly unbiased. +- **n > 100:** MLE becomes competitive and provides standard errors via Fisher Information, enabling confidence intervals and hypothesis tests. +- **Skewed distributions:** L-moments substantially outperform MOM, because conventional skewness estimates are highly variable for small samples. +- **US flood frequency analysis:** L-moments are recommended by USGS Bulletin 17C [[1]](#1). The Expected Moments Algorithm (EMA) extends the framework to handle censored and historical data. +- **Model selection:** When comparing candidate distributions, MLE enables the use of information criteria (AIC, BIC) for objective model ranking. + ## Distribution-Specific Estimation ### Log-Pearson Type III (USGS Bulletin 17C) @@ -371,107 +552,134 @@ For simpler distributions: ```cs double[] data = { 10.5, 12.3, 11.8, 15.2, 13.7, 14.1, 16.8, 12.9 }; -// Exponential - one parameter +// Exponential - two parameters (location + scale) var exponential = new Exponential(); exponential.Estimate(data, ParameterEstimationMethod.MethodOfMoments); -Console.WriteLine($"Exponential λ = {exponential.Lambda:F4}"); +Console.WriteLine($"Exponential ξ = {exponential.Xi:F4}, α = {exponential.Alpha:F4}"); // Log-Normal - two parameters var lognormal = new LogNormal(); lognormal.Estimate(data, ParameterEstimationMethod.MethodOfMoments); Console.WriteLine($"LogNormal μ = {lognormal.Mu:F4}, σ = {lognormal.Sigma:F4}"); -// Weibull - two parameters +// Weibull - two parameters (MLE only) var weibull = new Weibull(); -weibull.Estimate(data, ParameterEstimationMethod.MethodOfLinearMoments); -Console.WriteLine($"Weibull α = {weibull.Alpha:F4}, β = {weibull.Beta:F4}"); +weibull.Estimate(data, ParameterEstimationMethod.MaximumLikelihood); +Console.WriteLine($"Weibull λ = {weibull.Lambda:F4}, κ = {weibull.Kappa:F4}"); ``` -## Practical Workflow Example +## Tutorial: Complete Flood Frequency Analysis -Complete workflow for flood frequency analysis: +This tutorial demonstrates a complete distribution fitting workflow using real streamflow data from the White River near Nora, Indiana. The data and expected results are drawn from published references [[4]](#4) and validated against the R `lmom` package [[2]](#2). + +**Data source:** Rao, A. R. & Hamed, K. H. (2000). *Flood Frequency Analysis*. CRC Press, Table 7.1.2. +See also: [`example-data/white-river-nora-floods.csv`](../example-data/white-river-nora-floods.csv) ```cs using Numerics.Distributions; using Numerics.Data.Statistics; -// Step 1: Load and prepare data -double[] annualPeakFlows = LoadFloodData(); // Your data loading function -Console.WriteLine($"Sample size: {annualPeakFlows.Length}"); -Console.WriteLine($"Sample mean: {annualPeakFlows.Average():F0}"); -Console.WriteLine($"Sample std dev: {Statistics.StandardDeviation(annualPeakFlows):F0}"); +// White River near Nora, Indiana — 62 years of annual peak streamflow (cfs) +// Source: Rao & Hamed (2000), Table 7.1.2 +double[] annualPeaks = { + 23200, 2950, 10300, 23200, 4540, 9960, 10800, 26900, 23300, 20400, + 8480, 3150, 9380, 32400, 20800, 11100, 7270, 9600, 14600, 14300, + 22500, 14700, 12700, 9740, 3050, 8830, 12000, 30400, 27000, 15200, + 8040, 11700, 20300, 22700, 30400, 9180, 4870, 14700, 12800, 13700, + 7960, 9830, 12500, 10700, 13200, 14700, 14300, 4050, 14600, 14400, + 19200, 7160, 12100, 8650, 10600, 24500, 14400, 6300, 9560, 15800, + 14300, 28700 +}; + +Console.WriteLine($"Record length: {annualPeaks.Length} years"); +Console.WriteLine($"Range: {annualPeaks.Min():F0} - {annualPeaks.Max():F0} cfs"); + +// Step 1: Compute sample L-moments +// L-moments are more robust than product moments for small to moderate samples. +// Validated against R lmom::samlmu() +double[] lMoments = Statistics.LinearMoments(annualPeaks); -// Step 2: Compute sample L-moments -double[] lMoments = Statistics.LinearMoments(annualPeakFlows); Console.WriteLine($"\nSample L-moments:"); -Console.WriteLine($" λ₁ = {lMoments[0]:F0}"); -Console.WriteLine($" λ₂ = {lMoments[1]:F0}"); -Console.WriteLine($" τ₃ = {lMoments[2]:F4}"); -Console.WriteLine($" τ₄ = {lMoments[3]:F4}"); +Console.WriteLine($" λ₁ (L-location): {lMoments[0]:F1}"); +Console.WriteLine($" λ₂ (L-scale): {lMoments[1]:F1}"); +Console.WriteLine($" τ₃ (L-skewness): {lMoments[2]:F4}"); +Console.WriteLine($" τ₄ (L-kurtosis): {lMoments[3]:F4}"); -// Step 3: Fit multiple candidate distributions +// Step 2: Fit candidate distributions using L-moments var candidates = new List<(string Name, IUnivariateDistribution Dist)> { - ("LP3", new LogPearsonTypeIII()), - ("GEV", new GeneralizedExtremeValue()), + ("LP3", new LogPearsonTypeIII()), + ("GEV", new GeneralizedExtremeValue()), ("Gumbel", new Gumbel()), - ("PIII", new PearsonTypeIII()) + ("PIII", new PearsonTypeIII()) }; foreach (var (name, dist) in candidates) { - dist.Estimate(annualPeakFlows, ParameterEstimationMethod.MethodOfLinearMoments); - - Console.WriteLine($"\n{name} fitted:"); - var paramNames = dist.ParameterNamesShortForm; - var paramValues = dist.GetParameters; + dist.Estimate(annualPeaks, ParameterEstimationMethod.MethodOfLinearMoments); + Console.WriteLine($"\n{name} fitted parameters:"); + var pNames = dist.ParameterNamesShortForm; + var pValues = dist.GetParameters; for (int i = 0; i < dist.NumberOfParameters; i++) + Console.WriteLine($" {pNames[i]} = {pValues[i]:F4}"); +} + +// Step 3: Compare estimation methods for GEV +// Textbook (Rao & Hamed, Example 7.1.1, p. 218) provides MOM results for comparison. +Console.WriteLine("\nGEV: Comparing estimation methods:"); +foreach (var method in new[] { + ParameterEstimationMethod.MethodOfLinearMoments, + ParameterEstimationMethod.MethodOfMoments, + ParameterEstimationMethod.MaximumLikelihood }) +{ + var gev = new GeneralizedExtremeValue(); + try + { + gev.Estimate(annualPeaks, method); + Console.WriteLine($" {method}: ξ={gev.Xi:F1}, α={gev.Alpha:F1}, κ={gev.Kappa:F4}"); + } + catch (Exception ex) { - Console.WriteLine($" {paramNames[i]} = {paramValues[i]:F4}"); + Console.WriteLine($" {method}: Failed — {ex.Message}"); } } -// Step 4: Compare at key quantiles -var testProbs = new double[] { 0.5, 0.9, 0.98, 0.99, 0.998 }; - -Console.WriteLine($"\nQuantile Comparison:"); -Console.WriteLine($"AEP | " + string.Join(" | ", candidates.Select(c => $"{c.Name,8}"))); -Console.WriteLine(new string('-', 60)); +// Step 4: Compute flood frequency curve +Console.WriteLine("\nFlood Frequency Curve (LP3, L-moments):"); +Console.WriteLine("T (years) | AEP | Discharge (cfs)"); +Console.WriteLine("----------|----------|----------------"); -foreach (var p in testProbs) +var lp3 = (LogPearsonTypeIII)candidates[0].Dist; +foreach (int T in new[] { 2, 5, 10, 25, 50, 100, 200, 500 }) { - double aep = 1 - p; - var quantiles = candidates.Select(c => c.Dist.InverseCDF(p)); - Console.WriteLine($"{aep:F3} | " + string.Join(" | ", quantiles.Select(q => $"{q,8:F0}"))); + double aep = 1.0 / T; + double Q = lp3.InverseCDF(1 - aep); + Console.WriteLine($"{T,9} | {aep,8:F5} | {Q,14:F0}"); } -// Step 5: Select best distribution (using GOF or judgment) -// See goodness-of-fit documentation for formal selection -var selectedDist = candidates[0].Dist; // e.g., LP3 for USGS applications +// Step 5: Compare candidate distributions at key return periods +Console.WriteLine("\nQuantile Comparison (cfs):"); +Console.Write(" AEP "); +foreach (var c in candidates) Console.Write($"| {c.Name,8} "); +Console.WriteLine(); +Console.WriteLine(new string('-', 55)); -// Step 6: Compute design floods -Console.WriteLine($"\n100-year flood: {selectedDist.InverseCDF(0.99):F0} cfs"); -Console.WriteLine($"500-year flood: {selectedDist.InverseCDF(0.998):F0} cfs"); -``` - -## Estimation with Censored Data - -For data with detection limits or censoring: - -```cs -// Low flows below detection limit (left-censored) -double detectionLimit = 5.0; -var observed = data.Where(x => x >= detectionLimit).ToArray(); -int nCensored = data.Length - observed.Length; - -Console.WriteLine($"Observed: {observed.Length}, Censored: {nCensored}"); - -// Fit using only observed values -var lognormal = new LogNormal(); -lognormal.Estimate(observed, ParameterEstimationMethod.MethodOfMoments); +foreach (double p in new[] { 0.5, 0.9, 0.98, 0.99, 0.998 }) +{ + Console.Write($" {1 - p:F3} "); + foreach (var c in candidates) + Console.Write($"| {c.Dist.InverseCDF(p),8:F0} "); + Console.WriteLine(); +} -// Note: This is a simple approach. For formal censored data analysis, -// use MLE with censored likelihood (requires custom implementation) +// Step 6: Model selection using AIC +Console.WriteLine("\nModel Selection (AIC):"); +foreach (var (name, dist) in candidates) +{ + double logLik = dist.LogLikelihood(annualPeaks); + double aic = GoodnessOfFit.AIC(dist.NumberOfParameters, logLik); + Console.WriteLine($" {name,-8}: AIC = {aic:F1}"); +} ``` ## Tips and Best Practices @@ -479,11 +687,12 @@ lognormal.Estimate(observed, ParameterEstimationMethod.MethodOfMoments); ### 1. Sample Size Requirements ```cs -// Rule of thumb: n > 10 * number of parameters -if (data.Length < 3 * dist.NumberOfParameters * 10) +// Rule of thumb: n > 10 * number of parameters for MLE +int minSamples = 10 * dist.NumberOfParameters; +if (data.Length < minSamples) { - Console.WriteLine("Warning: Small sample size relative to parameters"); - Console.WriteLine("Consider using L-moments for improved efficiency"); + Console.WriteLine($"Warning: Sample size ({data.Length}) below recommended minimum ({minSamples})"); + Console.WriteLine("Consider using L-moments for improved small-sample efficiency"); } ``` @@ -527,41 +736,62 @@ foreach (var method in methods) ### 4. Outlier Detection +The Numerics library provides two formal statistical tests for detecting outliers in flood frequency data, both based on the Grubbs-Beck framework. + +#### Grubbs-Beck Test + +The original Grubbs-Beck test ([Grubbs & Beck, 1972][6]) identifies high and low outlier thresholds at the 10% significance level. The test assumes the log-transformed data follow a normal distribution and uses a critical value $K_n$ based on sample size: + +```math +X_{Hi} = \exp(\bar{y} + K_n \cdot s_y), \quad X_{Lo} = \exp(\bar{y} - K_n \cdot s_y) +``` + +where $\bar{y}$ and $s_y$ are the mean and standard deviation of the log-transformed sample. + ```cs -// Identify potential outliers before estimation -double[] sorted = data.OrderBy(x => x).ToArray(); -double Q1 = Statistics.Quantile(sorted, 0.25); -double Q3 = Statistics.Quantile(sorted, 0.75); -double IQR = Q3 - Q1; +using Numerics.Data.Statistics; -var outliers = data.Where(x => x < Q1 - 1.5 * IQR || x > Q3 + 1.5 * IQR).ToArray(); +double[] annualPeaks = { 50, 180, 220, 310, 450, 520, 680, 720, 890, 1050, 1200, 5200 }; -if (outliers.Length > 0) -{ - Console.WriteLine($"Potential outliers detected: {outliers.Length}"); - Console.WriteLine("Consider using L-moments (more robust to outliers)"); -} +// Original Grubbs-Beck test at 10% significance +MultipleGrubbsBeckTest.GrubbsBeckTest(annualPeaks, out double XHi, out double XLo); + +Console.WriteLine($"High outlier threshold: {XHi:F0} (values above are high outliers)"); +Console.WriteLine($"Low outlier threshold: {XLo:F0} (values below are low outliers)"); + +// Identify outliers +var highOutliers = annualPeaks.Where(x => x > XHi).ToArray(); +var lowOutliers = annualPeaks.Where(x => x < XLo).ToArray(); +Console.WriteLine($"High outliers: {highOutliers.Length}, Low outliers: {lowOutliers.Length}"); ``` -### 5. Historical Information +#### Multiple Grubbs-Beck Test -When historical data exists outside the systematic record: +The Multiple Grubbs-Beck Test ([Cohn et al., 2013][7]) is a generalization that can detect multiple potentially influential low flows (PILFs). This is particularly important in Bulletin 17C flood frequency analysis, where low outliers can distort the fitted distribution and bias upper-tail quantile estimates. ```cs -// Combine systematic record with historical peaks -double[] systematicRecord = { 12.5, 15.3, 11.2, 18.7, 14.1 }; // Recent, complete -double[] historicalPeaks = { 22.3 }; // Known historical floods +using Numerics.Data.Statistics; -// This is a simplified approach -// For formal analysis, use Expected Moments Algorithm (EMA) -var combined = systematicRecord.Concat(historicalPeaks).ToArray(); +double[] annualPeaks = { 5, 12, 18, 180, 220, 310, 450, 520, 680, 720, 890, 1050 }; -var lp3 = new LogPearsonTypeIII(); -lp3.Estimate(combined, ParameterEstimationMethod.MethodOfLinearMoments); +// Multiple Grubbs-Beck test — returns count of low outliers +int numLowOutliers = MultipleGrubbsBeckTest.Function(annualPeaks); -Console.WriteLine("Fitted with historical information included"); +Console.WriteLine($"Number of low outliers detected: {numLowOutliers}"); + +if (numLowOutliers > 0) +{ + // Sort to identify which values are low outliers + double[] sorted = annualPeaks.OrderBy(x => x).ToArray(); + Console.WriteLine("Low outlier values:"); + for (int i = 0; i < numLowOutliers; i++) + Console.WriteLine($" {sorted[i]}"); + Console.WriteLine("Consider censoring these values in LP3 frequency analysis"); +} ``` +> **Guidance:** Use the original Grubbs-Beck test for general-purpose outlier screening. Use the Multiple Grubbs-Beck test specifically for Bulletin 17C flood frequency analysis where multiple low outliers may influence the Log-Pearson Type III fit. + ## Common Pitfalls 1. **Using MOM for small samples** - Use L-moments instead @@ -569,7 +799,7 @@ Console.WriteLine("Fitted with historical information included"); 3. **Not checking parameter validity** - Always validate after estimation 4. **Wrong distribution family** - Use L-moment diagrams for selection 5. **Ignoring outliers** - L-moments are more robust than MOM or MLE -6. **Insufficient sample size** - Need at least 10n observations where n is number of parameters +6. **Insufficient sample size** - Need at least 10× the number of parameters (e.g., 30 samples for a 3-parameter distribution) --- @@ -581,6 +811,14 @@ Console.WriteLine("Fitted with historical information included"); [3] Mood, A. M., Graybill, F. A., & Boes, D. C. (1974). *Introduction to the Theory of Statistics* (3rd ed.). McGraw-Hill. +[4] Rao, A. R., & Hamed, K. H. (2000). *Flood Frequency Analysis*. CRC Press. + +[5] Casella, G. & Berger, R. L. (2002). *Statistical Inference* (2nd ed.). Duxbury/Thomson. + +[6] Grubbs, F. E., & Beck, G. (1972). Extension of sample sizes and percentage points for significance tests of outlying observations. *Technometrics*, 14(4), 847-854. + +[7] Cohn, T. A., England, J. F., Berenbrock, C. E., Mason, R. R., Stedinger, J. R., & Lamontagne, J. R. (2013). A generalized Grubbs-Beck test statistic for detecting multiple potentially influential low outliers in flood series. *Water Resources Research*, 49(8), 5047-5058. + --- [← Previous: Univariate Distributions](univariate.md) | [Back to Index](../index.md) | [Next: Uncertainty Analysis →](uncertainty-analysis.md) diff --git a/docs/distributions/uncertainty-analysis.md b/docs/distributions/uncertainty-analysis.md index cf1601a8..9aba68ab 100644 --- a/docs/distributions/uncertainty-analysis.md +++ b/docs/distributions/uncertainty-analysis.md @@ -6,18 +6,57 @@ Uncertainty analysis quantifies the confidence in estimated distribution paramet ## Bootstrap Analysis Overview -Bootstrap resampling [[1]](#1) is a nonparametric method for estimating sampling distributions and confidence intervals. It works by: +Bootstrap resampling [[1]](#1) is a powerful method for estimating sampling distributions and confidence intervals. The core idea is deceptively simple: rather than deriving the sampling distribution of an estimator analytically, approximate it empirically by repeatedly resampling and re-estimating. -1. Resampling the original data with replacement -2. Fitting the distribution to each bootstrap sample -3. Computing statistics from the ensemble of fitted distributions -4. Using percentiles of the bootstrap distribution for confidence intervals +The ***Numerics*** library implements a **parametric bootstrap**, which generates new samples from the fitted distribution rather than resampling the original observations with replacement. This is the natural approach when working with fitted probability distributions, and is particularly effective for extreme value analysis where the fitted distribution captures tail behavior that the limited observed data cannot. + +### The Parametric Bootstrap Algorithm + +Given observed data $x_1, x_2, \ldots, x_n$ and a fitted distribution $\hat{F}$ with estimated parameters $\hat{\theta}$, the parametric bootstrap proceeds as follows: + +**Step 1.** Generate $B$ bootstrap samples, each of size $n$, by drawing from the fitted distribution: + +```math +x^{*}_{b,1}, x^{*}_{b,2}, \ldots, x^{*}_{b,n} \sim \hat{F}(\hat{\theta}), \quad b = 1, 2, \ldots, B +``` + +**Step 2.** For each bootstrap sample $b$, re-estimate the distribution parameters using the same estimation method (e.g., L-moments, MLE) to obtain $\hat{\theta}^{*}_b$. + +**Step 3.** Compute the statistic of interest from each refitted distribution. For quantile estimation at non-exceedance probability $p$: + +```math +\hat{Q}^{*}_b = \hat{F}^{-1}(p \mid \hat{\theta}^{*}_b), \quad b = 1, 2, \ldots, B +``` + +**Step 4.** The empirical distribution of $`\hat{Q}^{*}_1, \hat{Q}^{*}_2, \ldots, \hat{Q}^{*}_B`$ approximates the sampling distribution of the quantile estimator $\hat{Q}$. + +From this bootstrap distribution, we can compute several useful summaries. The **bootstrap standard error** is the sample standard deviation of the bootstrap replicates: + +```math +\widehat{SE}_{boot} = \sqrt{\frac{1}{B-1}\sum_{b=1}^{B}\left(\hat{Q}^{*}_b - \bar{Q}^{*}\right)^2} +``` + +where $`\bar{Q}^{*} = \frac{1}{B}\sum_{b=1}^{B}\hat{Q}^{*}_b`$ is the mean of the bootstrap replicates. The **bootstrap estimate of bias** is: + +```math +\widehat{bias} = \bar{Q}^{*} - \hat{Q} +``` + +where $\hat{Q}$ is the original point estimate from the parent distribution. + +### Why Parametric Bootstrap? + +The parametric bootstrap is preferred over the nonparametric bootstrap for flood frequency analysis and similar applications because: +- It leverages the assumed distributional form to generate realistic samples, including plausible extreme values +- It produces smoother confidence intervals in the tails where data are sparse +- It naturally handles the small sample sizes typical of annual peak flow records +- It maintains consistency with the fitted distribution model used for design The bootstrap is particularly valuable when: -- Analytical confidence intervals are unavailable -- Sample sizes are small to moderate -- Distribution of estimators is unknown or complex -- Dealing with extreme value distributions +- Analytical confidence intervals are unavailable or intractable +- Sample sizes are small to moderate (common in hydrology) +- The sampling distribution of the estimator is unknown or complex +- Dealing with extreme value distributions where tail uncertainty is critical ## Creating a Bootstrap Analysis @@ -96,13 +135,13 @@ The results object contains: UnivariateDistributionBase ParentDistribution // Mode curve: quantiles from parent distribution -double[,] ModeCurve // [probability, quantile] +double[] ModeCurve // ModeCurve[i] = quantile at probabilities[i] // Mean curve: expected quantiles from bootstrap ensemble -double[,] MeanCurve // [probability, quantile] +double[] MeanCurve // MeanCurve[i] = expected quantile at probabilities[i] // Confidence intervals for quantiles -double[,] ConfidenceIntervals // [probability, lower, upper] +double[,] ConfidenceIntervals // [i, 0] = lower, [i, 1] = upper // Bootstrap parameter sets (if recorded) ParameterSet[] ParameterSets @@ -128,10 +167,10 @@ for (int i = 0; i < probabilities.Length; i++) double aep = 1 - prob; double T = 1.0 / aep; - double mode = results.ModeCurve[i, 1]; // Point estimate - double mean = results.MeanCurve[i, 1]; // Expected value - double lower = results.ConfidenceIntervals[i, 1]; // Lower bound - double upper = results.ConfidenceIntervals[i, 2]; // Upper bound + double mode = results.ModeCurve[i]; // Point estimate + double mean = results.MeanCurve[i]; // Expected value + double lower = results.ConfidenceIntervals[i, 0]; // Lower bound + double upper = results.ConfidenceIntervals[i, 1]; // Upper bound Console.WriteLine($"{aep:F3} | {T,9:F1} | {mode,8:F0} | {mean,8:F0} | [{lower,6:F0}, {upper,6:F0}]"); } @@ -157,10 +196,10 @@ using (var writer = new System.IO.StreamWriter("frequency_curve.csv")) double T = 1.0 / aep; writer.WriteLine($"{p:F4},{aep:F6},{T:F2}," + - $"{results.ModeCurve[i, 1]:F2}," + - $"{results.MeanCurve[i, 1]:F2}," + - $"{results.ConfidenceIntervals[i, 1]:F2}," + - $"{results.ConfidenceIntervals[i, 2]:F2}"); + $"{results.ModeCurve[i]:F2}," + + $"{results.MeanCurve[i]:F2}," + + $"{results.ConfidenceIntervals[i, 0]:F2}," + + $"{results.ConfidenceIntervals[i, 1]:F2}"); } } @@ -202,18 +241,26 @@ for (int j = 0; j < gev.NumberOfParameters; j++) Console.WriteLine($"\n{paramNames[j]}:"); Console.WriteLine($" Mean: {values.Average():F4}"); Console.WriteLine($" Std Dev: {Statistics.StandardDeviation(values.ToArray()):F4}"); - Console.WriteLine($" 5th percentile: {Statistics.Quantile(values.OrderBy(x => x).ToArray(), 0.05):F4}"); - Console.WriteLine($" 95th percentile: {Statistics.Quantile(values.OrderBy(x => x).ToArray(), 0.95):F4}"); + Console.WriteLine($" 5th percentile: {Statistics.Percentile(values.ToArray(), 0.05):F4}"); + Console.WriteLine($" 95th percentile: {Statistics.Percentile(values.ToArray(), 0.95):F4}"); } ``` ## Confidence Interval Methods -The ***Numerics*** library provides multiple methods for computing bootstrap confidence intervals, each with different properties. +The ***Numerics*** library provides five methods for computing bootstrap confidence intervals, each with different statistical properties. They range from simple (Percentile) to sophisticated (BCa, Bootstrap-t), trading off computational cost against accuracy of coverage. All methods construct a $(1-\alpha)\times 100\%$ confidence interval for a quantile $\hat{Q}$ at a given non-exceedance probability. ### 1. Percentile Method (Default) -The simplest method - uses percentiles of the bootstrap distribution: +The Percentile method [[1]](#1) is the simplest bootstrap confidence interval. It uses the quantiles of the bootstrap distribution directly as confidence limits: + +```math +CI_{1-\alpha} = \left[\hat{Q}^{*}_{(\alpha/2)},\;\hat{Q}^{*}_{(1-\alpha/2)}\right] +``` + +where $`\hat{Q}^{*}_{(p)}`$ denotes the $p$-th percentile of the bootstrap distribution $`\lbrace\hat{Q}^{*}_1, \ldots, \hat{Q}^{*}_B\rbrace`$. For a 90% confidence interval ($\alpha = 0.1$), this takes the 5th and 95th percentiles of the bootstrap replicates. + +The Percentile method is intuitive and easy to implement, but it does **not** correct for bias or skewness in the bootstrap distribution. It works well when the bootstrap distribution is approximately symmetric and the estimator is approximately unbiased. This is the default method used by the `Estimate()` method. ```cs var bootstrap = new BootstrapAnalysis(gev, @@ -236,7 +283,29 @@ for (int i = 0; i < probabilities.Length; i++) ### 2. Bias-Corrected (BC) Method -Corrects for bias in the bootstrap distribution: +The Bias-Corrected (BC) method [[2]](#2) adjusts the percentiles used for the confidence interval to correct for median bias in the bootstrap distribution. If the estimator is biased, the simple Percentile method produces intervals that are shifted; the BC method fixes this. + +First, compute the bias-correction factor $z_0$, which measures how far the bootstrap distribution median is from the original estimate: + +```math +z_0 = \Phi^{-1}\!\left(\frac{\#\{\hat{Q}^{*}_b \le \hat{Q}\}}{B + 1}\right) +``` + +where $\Phi^{-1}$ is the inverse of the standard Normal CDF, $\hat{Q}$ is the original point estimate, and the fraction counts the proportion of bootstrap replicates that fall at or below the original estimate. If the estimator is unbiased, approximately half the replicates will be below $\hat{Q}$, giving $z_0 \approx 0$. + +The adjusted percentiles for the confidence interval are: + +```math +\alpha_1 = \Phi\!\left(2z_0 + z_{\alpha/2}\right), \quad \alpha_2 = \Phi\!\left(2z_0 + z_{1-\alpha/2}\right) +``` + +where $z_{\alpha/2} = \Phi^{-1}(\alpha/2)$ is the standard Normal quantile. The confidence interval is then: + +```math +CI_{1-\alpha} = \left[\hat{Q}^{*}_{(\alpha_1)},\;\hat{Q}^{*}_{(\alpha_2)}\right] +``` + +When $z_0 = 0$ (no bias), this reduces to the standard Percentile method. The BC method is better than the Percentile method when the estimator has median bias, which is common for quantile estimators of skewed distributions. ```cs // Bias-corrected CI @@ -251,7 +320,25 @@ for (int i = 0; i < probabilities.Length; i++) ### 3. Normal Approximation Method -Assumes normal distribution for bootstrap statistics: +The Normal method assumes the bootstrap distribution of the statistic is approximately Gaussian. In ***Numerics***, this method applies a cube-root transformation to improve normality before computing the interval, then back-transforms the result: + +```math +\tilde{Q} = \hat{Q}^{1/3}, \quad \tilde{Q}^{*}_b = \left(\hat{Q}^{*}_b\right)^{1/3} +``` + +The bootstrap standard error is computed on the transformed scale: + +```math +\widetilde{SE} = \sqrt{\frac{1}{B-1}\sum_{b=1}^{B}\left(\tilde{Q}^{*}_b - \bar{\tilde{Q}}^{*}\right)^2} +``` + +The confidence interval is constructed in the transformed space and back-transformed: + +```math +CI_{1-\alpha} = \left[\left(\tilde{Q} + z_{\alpha/2}\cdot\widetilde{SE}\right)^3,\;\left(\tilde{Q} + z_{1-\alpha/2}\cdot\widetilde{SE}\right)^3\right] +``` + +where $z_{\alpha/2} = \Phi^{-1}(\alpha/2)$ is the standard Normal quantile. The cube-root transformation is a variance-stabilizing power transformation that makes the method approximately transformation-invariant, improving performance for skewed distributions common in hydrology. ```cs // Normal approximation CI @@ -266,7 +353,27 @@ for (int i = 0; i < probabilities.Length; i++) ### 4. BCa (Bias-Corrected and Accelerated) -The most accurate but computationally intensive method [[2]](#2): +The BCa method [[2]](#2) extends the BC method by adding an **acceleration constant** $\hat{a}$ that corrects for skewness in the bootstrap distribution. This makes the BCa interval **second-order accurate** and **transformation-respecting** -- meaning it gives the same answer regardless of what monotone transformation is applied to the data. + +The bias-correction factor $z_0$ is computed the same way as in the BC method. The acceleration constant $\hat{a}$ is estimated using the jackknife. For each observation $i = 1, \ldots, n$, the distribution is re-estimated with that observation removed and the statistic is recomputed, yielding jackknife replicates $\hat{Q}_{(i)}$. The acceleration constant is: + +```math +\hat{a} = \frac{\sum_{i=1}^{n}\left(\hat{Q} - \hat{Q}_{(i)}\right)^3}{6\left[\sum_{i=1}^{n}\left(\hat{Q} - \hat{Q}_{(i)}\right)^2\right]^{3/2}} +``` + +where $\hat{Q}$ is the original point estimate and $\hat{Q}_{(i)}$ is the estimate computed from the sample with observation $i$ removed. The acceleration constant measures the rate at which the standard error of $\hat{Q}$ changes as the true parameter value changes. + +The adjusted percentiles incorporate both bias correction and acceleration: + +```math +\alpha_1 = \Phi\!\left(z_0 + \frac{z_0 + z_{\alpha/2}}{1 - \hat{a}(z_0 + z_{\alpha/2})}\right), \quad \alpha_2 = \Phi\!\left(z_0 + \frac{z_0 + z_{1-\alpha/2}}{1 - \hat{a}(z_0 + z_{1-\alpha/2})}\right) +``` + +The confidence interval is then $`CI_{1-\alpha} = [\hat{Q}^{*}_{(\alpha_1)}, \hat{Q}^{*}_{(\alpha_2)}]`$. + +When $\hat{a} = 0$, the BCa method reduces to the BC method. When both $z_0 = 0$ and $\hat{a} = 0$, it reduces to the Percentile method. The BCa method is the most accurate of the percentile-based methods, but it requires the original sample data and is computationally expensive due to the jackknife (which requires $n$ additional distribution fits). + +> **Note:** The `BCaQuantileCI` method requires the original sample data because it re-estimates the distribution parameters internally and performs the jackknife. This means it calls `Estimate()` on the distribution, which will update the distribution's parameters. ```cs // BCa method requires original sample data @@ -284,7 +391,23 @@ for (int i = 0; i < probabilities.Length; i++) ### 5. Bootstrap-t Method -Uses studentized bootstrap for improved coverage: +The Bootstrap-t (studentized bootstrap) method [[3]](#3) is the bootstrap analog of the Student-t confidence interval. Rather than using percentiles of the bootstrap distribution directly, it constructs a pivotal quantity by standardizing each bootstrap replicate by its own standard error. This approach typically achieves the most accurate coverage for location-type parameters. + +Like the Normal method, ***Numerics*** applies a cube-root transformation for variance stabilization. For each bootstrap replicate $b$, the method computes both the transformed quantile estimate $\tilde{Q}^{*}_b = (\hat{Q}^{*}_b)^{1/3}$ and its standard error $\widetilde{SE}^{*}_b$ (estimated via an inner bootstrap of 300 replications). The studentized statistic is: + +```math +t^{*}_b = \frac{\tilde{Q} - \tilde{Q}^{*}_b}{\widetilde{SE}^{*}_b}, \quad b = 1, \ldots, B +``` + +where $\tilde{Q} = \hat{Q}^{1/3}$ is the transformed original estimate. The confidence interval is constructed using the percentiles of the studentized distribution and the overall bootstrap standard error $\widetilde{SE}$: + +```math +CI_{1-\alpha} = \left[\left(\tilde{Q} + t^{*}_{(\alpha/2)}\cdot\widetilde{SE}\right)^3,\;\left(\tilde{Q} + t^{*}_{(1-\alpha/2)}\cdot\widetilde{SE}\right)^3\right] +``` + +where $`t^{*}_{(p)}`$ is the $p$-th percentile of $`\lbrace t^{*}_1, \ldots, t^{*}_B\rbrace`$, and $\widetilde{SE}$ is the standard deviation of the transformed bootstrap replicates $`\lbrace\tilde{Q}^{*}_1, \ldots, \tilde{Q}^{*}_B\rbrace`$. + +The Bootstrap-t method is the most computationally expensive method because it requires a **double bootstrap**: each of the $B$ outer replications requires an inner bootstrap (300 replications by default) to estimate the standard error. However, it can provide the most accurate coverage probabilities for location parameters and is second-order accurate. ```cs // Bootstrap-t CI @@ -326,26 +449,33 @@ for (int i = 0; i < probabilities.Length; i++) ## Bootstrap Moments -Compute product moments and L-moments from bootstrap ensemble: +Compute product moments and L-moments from the bootstrap ensemble. Both methods return a `double[Replications, 4]` array where each row is one bootstrap replication and each column is a moment: ```cs -// Product moments from bootstrap replications +using Numerics.Data.Statistics; + +// Product moments: [replication, moment] +// Column 0 = mean, 1 = std dev, 2 = skewness, 3 = kurtosis double[,] productMoments = bootstrap.ProductMoments(); +int R = productMoments.GetLength(0); // number of replications + +// Summarize across replications for each moment Console.WriteLine("Bootstrap Product Moments:"); -Console.WriteLine($"Mean of means: {productMoments[0, 0]:F2}"); -Console.WriteLine($"Mean of std devs: {productMoments[1, 0]:F2}"); -Console.WriteLine($"Mean of skewness: {productMoments[2, 0]:F4}"); -Console.WriteLine($"Mean of kurtosis: {productMoments[3, 0]:F4}"); +Console.WriteLine($"Mean of means: {Enumerable.Range(0, R).Average(i => productMoments[i, 0]):F2}"); +Console.WriteLine($"Mean of std devs: {Enumerable.Range(0, R).Average(i => productMoments[i, 1]):F2}"); +Console.WriteLine($"Mean of skewness: {Enumerable.Range(0, R).Average(i => productMoments[i, 2]):F4}"); +Console.WriteLine($"Mean of kurtosis: {Enumerable.Range(0, R).Average(i => productMoments[i, 3]):F4}"); -// L-moments from bootstrap replications +// L-moments: [replication, moment] +// Column 0 = λ₁, 1 = λ₂, 2 = τ₃, 3 = τ₄ double[,] lMoments = bootstrap.LinearMoments(); Console.WriteLine("\nBootstrap L-Moments:"); -Console.WriteLine($"Mean λ₁: {lMoments[0, 0]:F2}"); -Console.WriteLine($"Mean λ₂: {lMoments[1, 0]:F2}"); -Console.WriteLine($"Mean τ₃: {lMoments[2, 0]:F4}"); -Console.WriteLine($"Mean τ₄: {lMoments[3, 0]:F4}"); +Console.WriteLine($"Mean λ₁: {Enumerable.Range(0, R).Average(i => lMoments[i, 0]):F2}"); +Console.WriteLine($"Mean λ₂: {Enumerable.Range(0, R).Average(i => lMoments[i, 1]):F2}"); +Console.WriteLine($"Mean τ₃: {Enumerable.Range(0, R).Average(i => lMoments[i, 2]):F4}"); +Console.WriteLine($"Mean τ₄: {Enumerable.Range(0, R).Average(i => lMoments[i, 3]):F4}"); ``` ## Expected Probability (Rare Events) @@ -429,10 +559,10 @@ for (int i = 0; i < returnPeriods.Length; i++) { int T = returnPeriods[i]; double aep = 1.0 / T; - double point = results.ModeCurve[i, 1]; - double mean = results.MeanCurve[i, 1]; - double lower = results.ConfidenceIntervals[i, 1]; - double upper = results.ConfidenceIntervals[i, 2]; + double point = results.ModeCurve[i]; + double mean = results.MeanCurve[i]; + double lower = results.ConfidenceIntervals[i, 0]; + double upper = results.ConfidenceIntervals[i, 1]; Console.WriteLine($"{T,6} {aep,11:F5} {point,8:F0} {mean,8:F0} {lower,8:F0} {upper,8:F0}"); } @@ -455,8 +585,8 @@ Console.WriteLine($"γ: {gammaValues.Average():F4} ± {Statistics.StandardDevia Console.WriteLine($"\nUncertainty Analysis Summary:"); for (int i = 0; i < returnPeriods.Length; i++) { - double width = results.ConfidenceIntervals[i, 2] - results.ConfidenceIntervals[i, 1]; - double relativeWidth = width / results.ModeCurve[i, 1] * 100; + double width = results.ConfidenceIntervals[i, 1] - results.ConfidenceIntervals[i, 0]; + double relativeWidth = width / results.ModeCurve[i] * 100; Console.WriteLine($"{returnPeriods[i]}-year: ±{relativeWidth:F1}% relative uncertainty"); } ``` @@ -485,31 +615,51 @@ var results2 = bootstrap.Estimate(probs2, alpha: 0.1, distributions: bootstrapDi ### Custom Quantile Computations +The `Quantiles()` method returns a `double[Replications, probabilities.Count]` array containing the raw quantile value from each bootstrap replication at each probability. You can then compute your own summary statistics: + ```cs +using Numerics.Data.Statistics; + // Compute quantiles from bootstrap ensemble var probsOfInterest = new double[] { 0.9, 0.95, 0.98, 0.99, 0.998 }; +// Returns double[Replications, probabilities.Count] double[,] quantiles = bootstrap.Quantiles(probsOfInterest); +int R = quantiles.GetLength(0); // number of replications + Console.WriteLine("Bootstrap Quantiles:"); Console.WriteLine("Prob | Mean | Std Dev | 5th %ile | 95th %ile"); Console.WriteLine("------------------------------------------------------------"); for (int i = 0; i < probsOfInterest.Length; i++) { - Console.WriteLine($"{probsOfInterest[i]:F3} | {quantiles[i, 0],9:F0} | " + - $"{quantiles[i, 1],9:F0} | {quantiles[i, 2],9:F0} | {quantiles[i, 3],9:F0}"); + // Extract column for this probability across all replications + double[] values = Enumerable.Range(0, R) + .Select(r => quantiles[r, i]) + .Where(v => !double.IsNaN(v)) + .ToArray(); + + double mean = values.Average(); + double sd = Statistics.StandardDeviation(values); + double p05 = Statistics.Percentile(values, 0.05); + double p95 = Statistics.Percentile(values, 0.95); + + Console.WriteLine($"{probsOfInterest[i]:F3} | {mean,9:F0} | {sd,9:F0} | {p05,9:F0} | {p95,9:F0}"); } ``` ### Computing Probabilities -Reverse direction - find probabilities for given quantiles: +Reverse direction — find CDF probabilities for given quantile values. The `Probabilities()` method returns a `double[Replications, quantiles.Count]` array of raw CDF values from each replication: ```cs var designFlows = new double[] { 15000, 20000, 25000, 30000 }; -double[,] probabilities = bootstrap.Probabilities(designFlows); +// Returns double[Replications, quantiles.Count] +double[,] probs = bootstrap.Probabilities(designFlows); + +int R = probs.GetLength(0); Console.WriteLine("Probabilities for Design Flows:"); Console.WriteLine("Flow | Mean Prob | Std Dev | 5th %ile | 95th %ile"); @@ -517,11 +667,21 @@ Console.WriteLine("--------------------------------------------------------"); for (int i = 0; i < designFlows.Length; i++) { - double meanAEP = 1 - probabilities[i, 0]; + // Extract column for this quantile across all replications + double[] values = Enumerable.Range(0, R) + .Select(r => probs[r, i]) + .Where(v => !double.IsNaN(v)) + .ToArray(); + + double meanProb = values.Average(); + double sd = Statistics.StandardDeviation(values); + double p05 = Statistics.Percentile(values, 0.05); + double p95 = Statistics.Percentile(values, 0.95); + + double meanAEP = 1 - meanProb; double meanT = 1.0 / meanAEP; - - Console.WriteLine($"{designFlows[i],5:F0} | {probabilities[i, 0],9:F4} | " + - $"{probabilities[i, 1],9:F4} | {probabilities[i, 2],9:F4} | {probabilities[i, 3],9:F4}"); + + Console.WriteLine($"{designFlows[i],5:F0} | {meanProb,9:F4} | {sd,9:F4} | {p05,9:F4} | {p95,9:F4}"); Console.WriteLine($" | T={meanT:F1} years"); } ``` @@ -544,8 +704,8 @@ foreach (var nRep in replicationCounts) var result = boot.Estimate(new[] { testProb }, alpha: 0.1); - double estimate = result.ModeCurve[0, 1]; - double width = result.ConfidenceIntervals[0, 2] - result.ConfidenceIntervals[0, 1]; + double estimate = result.ModeCurve[0]; + double width = result.ConfidenceIntervals[0, 1] - result.ConfidenceIntervals[0, 0]; Console.WriteLine($"{nRep,12} | {estimate,15:F0} | {width,8:F0} | {nRep / 1000.0,4:F1}×"); } @@ -604,9 +764,9 @@ if (nFailed > bootstrapDists.Length * 0.05) ```cs // Report point estimate ± uncertainty -double point = results.ModeCurve[0, 1]; -double lower = results.ConfidenceIntervals[0, 1]; -double upper = results.ConfidenceIntervals[0, 2]; +double point = results.ModeCurve[0]; +double lower = results.ConfidenceIntervals[0, 0]; +double upper = results.ConfidenceIntervals[0, 1]; Console.WriteLine($"100-year flood: {point:F0} cfs"); Console.WriteLine($"90% CI: [{lower:F0}, {upper:F0}] cfs"); @@ -630,7 +790,7 @@ foreach (var n in new[] { 10, 20, 50, 100 }) ParameterEstimationMethod.MethodOfLinearMoments, n, 1000); var testResults = testBoot.Estimate(new[] { 0.99 }, alpha: 0.1); - double width = testResults.ConfidenceIntervals[0, 2] - testResults.ConfidenceIntervals[0, 1]; + double width = testResults.ConfidenceIntervals[0, 1] - testResults.ConfidenceIntervals[0, 0]; Console.WriteLine($"n={n,3}: CI width = {width:F0}"); } ``` @@ -643,6 +803,53 @@ foreach (var n in new[] { 10, 20, 50, 100 }) 4. **Ignoring small sample bias** - Bootstrap can't fix fundamental data limitations 5. **Overinterpreting precision** - CI width reflects sampling uncertainty only +## Rules of Thumb for Number of Replications + +The number of bootstrap replications $B$ directly affects the stability and precision of the results. The following guidelines apply: + +| Purpose | Minimum $B$ | Recommended $B$ | +|---|---|---| +| Standard error estimation | 200 | 1,000 | +| Confidence intervals | 1,000 | 10,000 | +| Precise CI endpoints | 5,000 | 20,000--50,000 | +| Extreme quantiles (99th percentile and beyond) | 10,000 | 50,000+ | + +For life-safety applications, use at least 10,000 replications for confidence intervals and verify convergence by comparing results at different replication counts. The ***Numerics*** library enforces a minimum of 100 replications and a minimum sample size of 10. + +## When Bootstrap Fails + +The bootstrap is not a universal solution. Practitioners working on safety-critical projects must be aware of these limitations: + +- **Very small samples ($n < 15$):** The bootstrap distribution is a poor approximation of the true sampling distribution because the fitted parametric model may itself be unreliable. Confidence intervals will be too narrow (overly optimistic). The ***Numerics*** library enforces $n \ge 10$. + +- **Heavy-tailed distributions:** When the underlying distribution has very heavy tails (e.g., GEV with large positive shape parameter), the bootstrap variance estimate can itself be highly variable. More replications are needed, and results should be interpreted cautiously. + +- **Dependent data:** The standard parametric bootstrap assumes that observations are independent and identically distributed (i.i.d.). For time series data with serial correlation, the bootstrap underestimates uncertainty. Block bootstrap or other specialized methods are needed for dependent data. + +- **Parameters near boundary of parameter space:** If the true parameter lies on or near the boundary of the parameter space (e.g., shape parameter near zero for GEV), the bootstrap distribution may be inconsistent. Some bootstrap replications may fail to converge, which is why the library allows up to 20 retries per replication. + +- **Model misspecification:** The parametric bootstrap inherits any bias from the assumed distributional form. If the wrong distribution family is used, the bootstrap confidence intervals may have poor coverage even with large $B$. + +## Confidence Interval Method Selection Guide + +Choosing the right confidence interval method involves balancing accuracy against computational cost: + +| Method | Strengths | Weaknesses | Best For | +|---|---|---|---| +| **Percentile** | Simplest, fastest | No bias/skewness correction | Quick estimates; symmetric, unbiased cases | +| **Normal** | Simple, uses cube-root transform | Assumes approximate normality | Distributions with near-Normal quantile estimators | +| **Bias-Corrected (BC)** | Corrects for median bias | Does not correct for skewness | Moderately biased estimators | +| **BCa** | Second-order accurate; handles bias and skewness | Expensive (jackknife); needs original data | Best general-purpose accuracy | +| **Bootstrap-t** | Most accurate for location parameters; second-order accurate | Most expensive (double bootstrap) | Location parameters; when coverage accuracy is critical | + +**Practical recommendations:** + +- For **routine analyses**, the Percentile method (default) is adequate and fastest. +- For **design-level analyses**, use BC or BCa for improved accuracy. +- For **critical infrastructure** where confidence interval coverage must be precise, consider BCa or Bootstrap-t with $B \ge 20{,}000$. +- When **computational cost matters**, the Normal method offers a good balance of speed and accuracy through its cube-root variance-stabilizing transform. +- When in doubt, **compare methods**: if all five methods give similar intervals, any method is adequate. If they disagree substantially, the BCa or Bootstrap-t results should be preferred. + --- ## References diff --git a/docs/distributions/univariate.md b/docs/distributions/univariate.md index 7725f2c5..b7726ea2 100644 --- a/docs/distributions/univariate.md +++ b/docs/distributions/univariate.md @@ -1,6 +1,6 @@ # Univariate Distributions -[← Back to Index](../index.md) | [Next: Parameter Estimation →](parameter-estimation.md) +[← Previous: Hypothesis Tests](../statistics/hypothesis-tests.md) | [Back to Index](../index.md) | [Next: Parameter Estimation →](parameter-estimation.md) The ***Numerics*** library provides over 40 univariate probability distributions for statistical analysis, risk assessment, and uncertainty quantification. All distributions implement a common interface with consistent methods for computing probability density functions (PDF), cumulative distribution functions (CDF), quantiles, and statistical moments. @@ -13,10 +13,10 @@ The ***Numerics*** library provides over 40 univariate probability distributions | **Normal** | μ (mean), σ (std dev) | General purpose, natural phenomena | | **Log-Normal** | μ, σ | Right-skewed data, multiplicative processes | | **Uniform** | a (min), b (max) | Maximum entropy, prior distributions | -| **Exponential** | λ (rate) | Time between events, survival analysis | -| **Gamma** | α (shape), β (scale) | Waiting times, rainfall | +| **Exponential** | ξ (location), α (scale) | Time between events, survival analysis | +| **Gamma** | θ (scale), κ (shape) | Waiting times, rainfall | | **Beta** | α, β | Probabilities, proportions, [0,1] bounded | -| **Weibull** | α (scale), β (shape) | Failure times, wind speed | +| **Weibull** | λ (scale), κ (shape) | Failure times, wind speed | | **Gumbel** | ξ (location), α (scale) | Extreme values (maxima) | | **Generalized Extreme Value (GEV)** | ξ, α, κ (shape) | Block maxima, floods, earthquakes | | **Generalized Pareto (GP)** | ξ, α, κ | Exceedances over threshold | @@ -37,7 +37,7 @@ The ***Numerics*** library provides over 40 univariate probability distributions | **Inverse Chi-Squared** | ν | Bayesian inference | | **Pareto** | xₘ (scale), α (shape) | Income distributions, city sizes | | **PERT** | a, b, c | Project management, expert judgment | -| **PERT Percentile** | P₁₀, P₅₀, P₉₀ | Expert percentile elicitation | +| **PERT Percentile** | P₅, P₅₀, P₉₅ | Expert percentile elicitation | | **PERT Percentile Z** | Similar to PERT Percentile | Alternative parametrization | | **Truncated Normal** | μ, σ, a, b | Bounded normal distributions | | **Truncated Distribution** | Any distribution + bounds | Bounded versions of distributions | @@ -46,6 +46,7 @@ The ***Numerics*** library provides over 40 univariate probability distributions | **Kernel Density** | Sample data, bandwidth | Smooth non-parametric estimation | | **Deterministic** | Single value | Point estimates, constants | | **Competing Risks** | Multiple distributions | Failure analysis with multiple causes | +| **Von Mises** | μ (mean direction), κ (concentration) | Circular data, flood seasonality | ### Discrete Distributions @@ -99,16 +100,16 @@ double[] GenerateRandomValues(int sampleSize, int seed = -1) using Numerics.Distributions; // Normal distribution: N(100, 15) -var normal = new Normal(mu: 100, sigma: 15); +var normal = new Normal(mean: 100, standardDeviation: 15); // Generalized Extreme Value: GEV(1000, 200, -0.1) -var gev = new GeneralizedExtremeValue(xi: 1000, alpha: 200, kappa: -0.1); +var gev = new GeneralizedExtremeValue(location: 1000, scale: 200, shape: -0.1); // Log-Normal distribution -var lognormal = new LogNormal(mu: 4.5, sigma: 0.5); +var lognormal = new LogNormal(meanOfLog: 4.5, standardDeviationOfLog: 0.5); // Gamma distribution -var gamma = new GammaDistribution(alpha: 5, beta: 2); +var gamma = new GammaDistribution(scale: 5, shape: 2); ``` ### Method 2: Using SetParameters @@ -116,10 +117,10 @@ var gamma = new GammaDistribution(alpha: 5, beta: 2); ```cs // Create with default parameters, then set var weibull = new Weibull(); -weibull.SetParameters(new double[] { 50, 2.5 }); // alpha=50, beta=2.5 +weibull.SetParameters(new double[] { 50, 2.5 }); // lambda=50, kappa=2.5 // Or use named parameters -weibull.SetParameters(alpha: 50, beta: 2.5); +weibull.SetParameters(scale: 50, shape: 2.5); ``` ### Method 3: From Parameter Array @@ -161,7 +162,7 @@ Console.WriteLine($"95th percentile: {q95:F2}"); // 124.67 ### Statistical Properties ```cs -var gev = new GeneralizedExtremeValue(xi: 1000, alpha: 200, kappa: -0.1); +var gev = new GeneralizedExtremeValue(location: 1000, scale: 200, shape: -0.1); Console.WriteLine($"Mean: {gev.Mean:F2}"); Console.WriteLine($"Std Dev: {gev.StandardDeviation:F2}"); @@ -176,13 +177,13 @@ Console.WriteLine($"Mode: {gev.Mode:F2}"); The hazard function describes instantaneous failure rate: ```cs -var weibull = new Weibull(alpha: 100, beta: 2.5); +var weibull = new Weibull(scale: 100, shape: 2.5); // Hazard at time t=50 double hazard = weibull.HF(50); Console.WriteLine($"Hazard rate at t=50: {hazard:F6}"); -// For Weibull, hazard increases with time when β > 1 (wear-out) +// For Weibull, hazard increases with time when κ > 1 (wear-out) ``` ### Log-Space Calculations @@ -201,14 +202,440 @@ Console.WriteLine($"CDF(-10) = {cdf:E10}"); Console.WriteLine($"Log-CDF(-10) = {logCDF:F4}"); ``` +## Distribution Mathematical Definitions + +This section provides the mathematical foundations for the most commonly used distributions in ***Numerics***. Understanding the underlying probability density functions (PDF), cumulative distribution functions (CDF), and moment structure is essential for correct application, particularly in safety-critical engineering analysis. + +### Normal Distribution + +The Normal (Gaussian) distribution is the most widely used continuous distribution. It arises naturally from the Central Limit Theorem, which states that the sum of many independent random variables tends toward a Normal distribution regardless of the underlying distributions. In ***Numerics***, the Normal distribution is parameterized by its mean $\mu$ and standard deviation $\sigma$. + +#### Mathematical Definition + +**Probability Density Function (PDF):** + +```math +f(x) = \frac{1}{\sigma\sqrt{2\pi}} \exp\!\left(-\frac{(x - \mu)^2}{2\sigma^2}\right), \quad -\infty < x < \infty +``` + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = \Phi\!\left(\frac{x - \mu}{\sigma}\right) = \frac{1}{2}\left[1 + \text{erf}\!\left(\frac{x - \mu}{\sigma\sqrt{2}}\right)\right] +``` + +where $\Phi(\cdot)$ is the standard Normal CDF and $\text{erf}(\cdot)$ is the error function. + +**Moments:** + +| Property | Formula | +|----------|---------| +| Mean | $E[X] = \mu$ | +| Variance | $\text{Var}(X) = \sigma^2$ | +| Skewness | $\gamma_1 = 0$ | +| Kurtosis | $\kappa = 3$ | + +**Parameters in Numerics:** `Normal(mean, standardDeviation)` where `mean` = $\mu$ and `standardDeviation` = $\sigma > 0$. + +```cs +using Numerics.Distributions; + +var normal = new Normal(mean: 100, standardDeviation: 15); + +double pdf = normal.PDF(110); // f(110) +double cdf = normal.CDF(110); // P(X <= 110) +double q95 = normal.InverseCDF(0.95); // 95th percentile + +Console.WriteLine($"Mean: {normal.Mean}"); // 100 +Console.WriteLine($"Std Dev: {normal.StandardDeviation}"); // 15 +Console.WriteLine($"Skewness: {normal.Skewness}"); // 0 +``` + +### Log-Normal Distribution + +A random variable $X$ follows a Log-Normal distribution if $\log(X)$ follows a Normal distribution. The Log-Normal distribution is appropriate for strictly positive, right-skewed data arising from multiplicative processes. In ***Numerics***, the Log-Normal is parameterized by the mean and standard deviation of the log-transformed data, and uses base-10 logarithms by default. The `Base` property can be changed to use natural logarithms or any other base. + +#### Mathematical Definition + +For a general logarithmic base $b$, let $K = 1/\ln(b)$. The PDF and CDF are expressed in terms of $\log_b(x)$. + +**Probability Density Function (PDF):** + +```math +f(x) = \frac{K}{x \sigma \sqrt{2\pi}} \exp\!\left(-\frac{(\log_b x - \mu)^2}{2\sigma^2}\right), \quad x > 0 +``` + +where $\mu$ and $\sigma$ are the mean and standard deviation of $\log_b(X)$. + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = \frac{1}{2}\left[1 + \text{erf}\!\left(\frac{\log_b x - \mu}{\sigma\sqrt{2}}\right)\right] +``` + +**Moments (general base $b$, where $\beta = \ln(b)$):** + +| Property | Formula | +|----------|---------| +| Mean | $E[X] = \exp\!\left[(\mu + \tfrac{1}{2}\sigma^2 \beta)\,\beta\right]$ | +| Mode | $\exp(\mu / K) = b^{\mu}$ | + +**Key relationship:** If $X \sim \text{LogNormal}(\mu, \sigma)$, then $\log_b(X) \sim \text{Normal}(\mu, \sigma)$. + +**Parameters in Numerics:** `LogNormal(meanOfLog, standardDeviationOfLog)` where `meanOfLog` = $\mu$ and `standardDeviationOfLog` = $\sigma > 0$. Default base is 10. + +```cs +// Log-Normal with base-10 parameters +var lognormal = new LogNormal(meanOfLog: 4.5, standardDeviationOfLog: 0.5); + +// Default base is 10 +Console.WriteLine($"Base: {lognormal.Base}"); // 10 + +// Switch to natural log if needed +lognormal.Base = Math.E; + +// Key relationship: log(X) ~ Normal +double x = lognormal.InverseCDF(0.5); +Console.WriteLine($"Median: {x:F2}"); +Console.WriteLine($"log10(Median): {Math.Log10(x):F2}"); // equals Mu when Base=10 +``` + +### Gamma Distribution + +The Gamma distribution is a flexible two-parameter family for modeling positive-valued random variables. It generalizes the Exponential distribution and appears frequently in waiting-time problems, rainfall modeling, and Bayesian statistics. The ***Numerics*** library uses the shape/scale parameterization. + +#### Mathematical Definition + +**Probability Density Function (PDF):** + +```math +f(x) = \frac{x^{\kappa-1}\, e^{-x/\theta}}{\theta^{\kappa}\,\Gamma(\kappa)}, \quad x > 0 +``` + +where $\theta > 0$ is the scale parameter and $\kappa > 0$ is the shape parameter. $\Gamma(\cdot)$ is the gamma function. + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = \frac{\gamma(\kappa,\, x/\theta)}{\Gamma(\kappa)} = P(\kappa,\, x/\theta) +``` + +where $\gamma(\kappa, z)$ is the lower incomplete gamma function and $P(\kappa, z)$ is the regularized lower incomplete gamma function. + +**Moments:** + +| Property | Formula | +|----------|---------| +| Mean | $E[X] = \kappa\theta$ | +| Variance | $\text{Var}(X) = \kappa\theta^2$ | +| Skewness | $\gamma_1 = 2/\sqrt{\kappa}$ | +| Kurtosis | $\kappa_4 = 3 + 6/\kappa$ | +| Mode | $(\kappa - 1)\theta$ for $\kappa \geq 1$ | + +**Special cases:** When $\kappa$ is a positive integer, the Gamma distribution is also known as the Erlang distribution. + +**Parameters in Numerics:** `GammaDistribution(scale, shape)` where `scale` = $\theta > 0$ and `shape` = $\kappa > 0$. + +```cs +var gamma = new GammaDistribution(scale: 5, shape: 2); + +Console.WriteLine($"Mean: {gamma.Mean}"); // κθ = 10 +Console.WriteLine($"Variance: {gamma.Variance}"); // κθ² = 50 +Console.WriteLine($"Skewness: {gamma.Skewness:F4}"); // 2/√κ +Console.WriteLine($"Rate (1/θ): {gamma.Rate}"); // 0.2 + +double cdf = gamma.CDF(15.0); +double quantile = gamma.InverseCDF(0.95); +``` + +### Exponential Distribution + +The Exponential distribution models the time between events in a Poisson process. Its defining property is *memorylessness*: the probability of an event occurring in the next $\Delta t$ time units is independent of how much time has already elapsed. In ***Numerics***, the Exponential distribution is a two-parameter (shifted) distribution with location $\xi$ and scale $\alpha$. Setting $\xi = 0$ yields the standard one-parameter Exponential. + +#### Mathematical Definition + +**Probability Density Function (PDF):** + +```math +f(x) = \frac{1}{\alpha}\exp\!\left(-\frac{x - \xi}{\alpha}\right), \quad x \geq \xi +``` + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = 1 - \exp\!\left(-\frac{x - \xi}{\alpha}\right), \quad x \geq \xi +``` + +**Inverse CDF (Quantile Function):** + +```math +Q(p) = \xi - \alpha \ln(1 - p) +``` + +**Moments:** + +| Property | Formula | +|----------|---------| +| Mean | $E[X] = \xi + \alpha$ | +| Variance | $\text{Var}(X) = \alpha^2$ | +| Skewness | $\gamma_1 = 2$ | +| Kurtosis | $\kappa = 9$ | +| Mode | $\xi$ | + +**Relationship to Gamma:** The Exponential($\xi$, $\alpha$) distribution is a special case of a shifted Gamma distribution with shape $\kappa = 1$. + +**Parameters in Numerics:** `Exponential(location, scale)` where `location` = $\xi$ and `scale` = $\alpha > 0$. A single-parameter constructor `Exponential(scale)` sets $\xi = 0$. + +```cs +// Two-parameter (shifted) exponential +var exp2 = new Exponential(location: 10, scale: 5); +Console.WriteLine($"Mean: {exp2.Mean}"); // ξ + α = 15 +Console.WriteLine($"Mode: {exp2.Mode}"); // ξ = 10 + +// One-parameter (standard) exponential +var exp1 = new Exponential(scale: 5); +Console.WriteLine($"Mean: {exp1.Mean}"); // α = 5 +Console.WriteLine($"P(X > 10): {exp1.CCDF(10):F4}"); // memoryless property +``` + +### Gumbel Distribution (Extreme Value Type I) + +The Gumbel distribution is a special case of the Generalized Extreme Value (GEV) distribution with shape parameter $\kappa = 0$. It models the distribution of the maximum (or minimum) of a sample drawn from various distributions with exponentially decaying tails. It is widely used in hydrology for modeling annual maximum floods and in structural engineering for modeling extreme wind loads. + +#### Mathematical Definition + +**Probability Density Function (PDF):** + +```math +f(x) = \frac{1}{\alpha}\exp\!\left[-(z + e^{-z})\right], \quad z = \frac{x - \xi}{\alpha}, \quad -\infty < x < \infty +``` + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = \exp\!\left(-e^{-z}\right), \quad z = \frac{x - \xi}{\alpha} +``` + +**Inverse CDF (Quantile Function):** + +```math +Q(p) = \xi - \alpha \ln(-\ln p) +``` + +**Moments:** + +| Property | Formula | +|----------|---------| +| Mean | $E[X] = \xi + \alpha\gamma_E$ where $\gamma_E \approx 0.5772$ is the Euler-Mascheroni constant | +| Variance | $\text{Var}(X) = \frac{\pi^2}{6}\alpha^2$ | +| Skewness | $\gamma_1 \approx 1.1396$ | +| Kurtosis | $\kappa = 3 + 12/5 = 5.4$ | +| Mode | $\xi$ | +| Median | $\xi - \alpha\ln(\ln 2)$ | + +**Parameters in Numerics:** `Gumbel(location, scale)` where `location` = $\xi$ and `scale` = $\alpha > 0$. + +```cs +var gumbel = new Gumbel(location: 100, scale: 25); + +Console.WriteLine($"Mode: {gumbel.Mode}"); // ξ = 100 +Console.WriteLine($"Mean: {gumbel.Mean:F2}"); // ξ + αγ ≈ 114.43 +Console.WriteLine($"Median: {gumbel.Median:F2}"); + +// 100-year return period quantile +double q100 = gumbel.InverseCDF(0.99); +Console.WriteLine($"1% AEP quantile: {q100:F2}"); +``` + +### Uniform Distribution + +The Uniform distribution assigns equal probability density to all values within a bounded interval $[a, b]$. It represents maximum uncertainty (maximum entropy) given only knowledge of the support bounds. It is commonly used as a non-informative prior in Bayesian analysis and for random number generation. + +#### Mathematical Definition + +**Probability Density Function (PDF):** + +```math +f(x) = \frac{1}{b - a}, \quad a \leq x \leq b +``` + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = \frac{x - a}{b - a}, \quad a \leq x \leq b +``` + +**Inverse CDF (Quantile Function):** + +```math +Q(p) = a + p(b - a) +``` + +**Moments:** + +| Property | Formula | +|----------|---------| +| Mean | $E[X] = \frac{a + b}{2}$ | +| Variance | $\text{Var}(X) = \frac{(b - a)^2}{12}$ | +| Skewness | $\gamma_1 = 0$ | +| Kurtosis | $\kappa = 9/5 = 1.8$ | + +**Parameters in Numerics:** `Uniform(min, max)` where `min` = $a$ and `max` = $b \geq a$. + +```cs +var uniform = new Uniform(min: 0, max: 10); + +Console.WriteLine($"Mean: {uniform.Mean}"); // 5 +Console.WriteLine($"Std Dev: {uniform.StandardDeviation:F4}"); // 10/√12 +Console.WriteLine($"PDF(5): {uniform.PDF(5)}"); // 0.1 everywhere in [0,10] + +double median = uniform.InverseCDF(0.5); // 5 +``` + +### Triangular Distribution + +The Triangular distribution is defined by three parameters: minimum $a$, mode $c$, and maximum $b$. It provides a simple model for uncertainty when only the range and most likely value are known. It is frequently used in risk assessment, project management (PERT analysis), and expert elicitation. + +#### Mathematical Definition + +**Probability Density Function (PDF):** + +```math +f(x) = \begin{cases} +\dfrac{2(x - a)}{(b - a)(c - a)} & a \leq x < c \\[6pt] +\dfrac{2}{b - a} & x = c \\[6pt] +\dfrac{2(b - x)}{(b - a)(b - c)} & c < x \leq b +\end{cases} +``` + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = \begin{cases} +\dfrac{(x - a)^2}{(b - a)(c - a)} & a \leq x \leq c \\[6pt] +1 - \dfrac{(b - x)^2}{(b - a)(b - c)} & c < x \leq b +\end{cases} +``` + +**Moments:** + +| Property | Formula | +|----------|---------| +| Mean | $E[X] = \frac{a + b + c}{3}$ | +| Variance | $\text{Var}(X) = \frac{a^2 + b^2 + c^2 - ab - ac - bc}{18}$ | +| Mode | $c$ | + +**Parameters in Numerics:** `Triangular(min, mode, max)` where `min` = $a$, `mode` = $c$, and `max` = $b$, with $a \leq c \leq b$. + +```cs +var tri = new Triangular(min: 10, mode: 15, max: 25); + +Console.WriteLine($"Mean: {tri.Mean:F2}"); // (10+25+15)/3 = 16.67 +Console.WriteLine($"Mode: {tri.Mode}"); // 15 +Console.WriteLine($"Median: {tri.Median:F2}"); +Console.WriteLine($"Std Dev: {tri.StandardDeviation:F2}"); +``` + +### Beta Distribution + +The Beta distribution is defined on the interval $[0, 1]$ and is parameterized by two positive shape parameters $\alpha$ and $\beta$. Its flexibility makes it ideal for modeling random proportions, probabilities, and percentages. In Bayesian statistics, it serves as the conjugate prior for the Bernoulli and Binomial distributions. + +#### Mathematical Definition + +**Probability Density Function (PDF):** + +```math +f(x) = \frac{x^{\alpha-1}(1-x)^{\beta-1}}{B(\alpha, \beta)}, \quad 0 \leq x \leq 1 +``` + +where $B(\alpha, \beta)$ is the Beta function: + +```math +B(\alpha, \beta) = \frac{\Gamma(\alpha)\,\Gamma(\beta)}{\Gamma(\alpha + \beta)} +``` + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = I_x(\alpha, \beta) +``` + +where $I_x(\alpha, \beta)$ is the regularized incomplete Beta function. + +**Moments:** + +| Property | Formula | +|----------|---------| +| Mean | $E[X] = \frac{\alpha}{\alpha + \beta}$ | +| Variance | $\text{Var}(X) = \frac{\alpha\beta}{(\alpha+\beta)^2(\alpha+\beta+1)}$ | +| Mode | $\frac{\alpha - 1}{\alpha + \beta - 2}$ for $\alpha, \beta > 1$ | +| Skewness | $\frac{2(\beta - \alpha)\sqrt{\alpha + \beta + 1}}{(\alpha + \beta + 2)\sqrt{\alpha\beta}}$ | + +**Special cases:** $\text{Beta}(1, 1) = \text{Uniform}(0, 1)$. + +**Parameters in Numerics:** `BetaDistribution(alpha, beta)` where `alpha` = $\alpha > 0$ and `beta` = $\beta > 0$. + +```cs +var beta = new BetaDistribution(alpha: 2, beta: 5); + +Console.WriteLine($"Mean: {beta.Mean:F4}"); // α/(α+β) = 2/7 +Console.WriteLine($"Mode: {beta.Mode:F4}"); // (α-1)/(α+β-2) = 1/5 +Console.WriteLine($"Variance: {beta.Variance:F4}"); + +// Beta(1,1) is equivalent to Uniform(0,1) +var betaUniform = new BetaDistribution(alpha: 1, beta: 1); +Console.WriteLine($"Beta(1,1) PDF(0.5): {betaUniform.PDF(0.5)}"); // 1.0 +``` + ## Hydrological Distributions ### Log-Pearson Type III (LP3) -The LP3 distribution is the standard for USGS flood frequency analysis [[1]](#1): +The LP3 distribution is the standard distribution for flood frequency analysis in the United States, as prescribed by Bulletin 17C [[1]](#1). It is constructed by applying the Pearson Type III distribution to the logarithms of the data. This means that if $X \sim \text{LP3}$, then $\log_b(X) \sim \text{Pearson Type III}$, where $b$ is the logarithmic base (default base 10 in ***Numerics***). + +#### Mathematical Definition + +The LP3 distribution is parameterized by the moments of the log-transformed data: $\mu$ (mean of log), $\sigma$ (standard deviation of log), and $\gamma$ (skewness of log). These parameters relate to the underlying Pearson Type III distribution through: + +| LP3 Parameter | Pearson Type III Equivalent | +|---------------|---------------------------| +| Location $\xi$ | $\mu - 2\sigma/\gamma$ | +| Scale $\beta$ | $\sigma\gamma/2$ | +| Shape $\alpha$ | $4/\gamma^2$ | + +The PDF and CDF of the LP3 are obtained by applying the Pearson Type III to the log-transformed variable. For a variable $Y = \log_b(X)$: + +**Pearson Type III PDF (applied to Y):** + +```math +f_Y(y) = \frac{|y - \xi|^{\alpha-1}\, \exp(-|y - \xi|/|\beta|)}{|\beta|^{\alpha}\,\Gamma(\alpha)} +``` + +where the sign conventions depend on the sign of $\gamma$: when $\gamma > 0$, $Y \geq \xi$; when $\gamma < 0$, $Y \leq \xi$. + +**Quantile computation:** In practice, LP3 quantiles are computed using the frequency factor approach from Bulletin 17C: + +```math +\log_b(X_p) = \mu + K_p \cdot \sigma +``` + +where $K_p$ is the Pearson Type III frequency factor (a function of the skewness $\gamma$ and probability $p$), computed via the Cornish-Fisher or Wilson-Hilferty approximation. + +**Moments (of the log-transformed data):** + +| Property | Formula | +|----------|---------| +| Mean of log | $\mu$ | +| Std dev of log | $\sigma$ | +| Skewness of log | $\gamma$ | + +When $\gamma = 0$, the LP3 reduces to the Log-Normal distribution. + +**Parameters in Numerics:** `LogPearsonTypeIII(meanOfLog, standardDeviationOfLog, skewOfLog)`. Default logarithmic base is 10. ```cs using Numerics.Distributions; +using Numerics.Data.Statistics; double[] annualPeakFlows = { 12500, 15300, 11200, 18700, 14100, 16800, 13400, 17200 }; @@ -234,7 +661,59 @@ Console.WriteLine($"500-year flood: {q500:F0} cfs"); ### Generalized Extreme Value (GEV) -GEV is widely used for extreme value analysis [[2]](#2): +The Generalized Extreme Value distribution unifies three classical extreme value distributions into a single three-parameter family [[2]](#2). It is the limiting distribution for block maxima (e.g., annual maximum floods, peak wind speeds) under very general conditions described by the Fisher-Tippett-Gnedenko theorem. + +#### Mathematical Definition + +The GEV is parameterized by location $\xi$, scale $\alpha > 0$, and shape $\kappa$. The ***Numerics*** library uses the Hosking parameterization where the sign convention for $\kappa$ follows L-moment theory. + +**Probability Density Function (PDF):** + +```math +f(x) = \frac{1}{\alpha} \exp\!\left[-(1-\kappa)y - e^{-y}\right] +``` + +where the reduced variate $y$ is defined as: + +```math +y = \begin{cases} +-\dfrac{1}{\kappa}\ln\!\left(1 - \kappa\dfrac{x - \xi}{\alpha}\right) & \kappa \neq 0 \\[6pt] +\dfrac{x - \xi}{\alpha} & \kappa = 0 +\end{cases} +``` + +**Cumulative Distribution Function (CDF):** + +```math +F(x) = \exp(-e^{-y}) +``` + +**Inverse CDF (Quantile Function):** + +```math +Q(p) = \begin{cases} +\xi + \dfrac{\alpha}{\kappa}\left[1 - (-\ln p)^{\kappa}\right] & \kappa \neq 0 \\[6pt] +\xi - \alpha \ln(-\ln p) & \kappa = 0 +\end{cases} +``` + +**Support:** + +| Shape | Sub-type | Upper/Lower Bound | +|-------|----------|-------------------| +| $\kappa = 0$ | Type I (Gumbel) | $-\infty < x < \infty$ | +| $\kappa > 0$ | Type III (Weibull) | $x \leq \xi + \alpha/\kappa$ (bounded upper tail) | +| $\kappa < 0$ | Type II (Frechet) | $x \geq \xi + \alpha/\kappa$ (bounded lower tail, heavy upper tail) | + +**Moments (exist when $|\kappa| < 1$ for the mean, $|\kappa| < 1/2$ for the variance):** + +| Property | Formula ($\kappa \neq 0$) | Formula ($\kappa = 0$, Gumbel) | +|----------|--------------------------|-------------------------------| +| Mean | $\xi + \frac{\alpha}{\kappa}[1 - \Gamma(1+\kappa)]$ | $\xi + \alpha\gamma_E$ | +| Variance | $\frac{\alpha^2}{\kappa^2}[\Gamma(1+2\kappa) - \Gamma^2(1+\kappa)]$ | $\frac{\pi^2}{6}\alpha^2$ | +| Mode | $\xi + \frac{\alpha}{\kappa}[(1+\kappa)^{-\kappa} - 1]$ | $\xi$ | + +**Parameters in Numerics:** `GeneralizedExtremeValue(location, scale, shape)` where `location` = $\xi$, `scale` = $\alpha > 0$, and `shape` = $\kappa$. ```cs // Annual maximum flood data @@ -249,9 +728,9 @@ Console.WriteLine($" Scale (α): {gev.Alpha:F2}"); Console.WriteLine($" Shape (κ): {gev.Kappa:F4}"); // Interpret shape parameter -if (gev.Kappa < 0) +if (gev.Kappa > 0) Console.WriteLine(" Type III (Weibull) - bounded upper tail"); -else if (gev.Kappa > 0) +else if (gev.Kappa < 0) Console.WriteLine(" Type II (Fréchet) - heavy upper tail"); else Console.WriteLine(" Type I (Gumbel) - exponential tail"); @@ -286,7 +765,7 @@ Create truncated versions of any distribution: ```cs // Normal truncated to [0, 100] -var truncNormal = new TruncatedNormal(mu: 50, sigma: 15, lowerBound: 0, upperBound: 100); +var truncNormal = new TruncatedNormal(mean: 50, standardDeviation: 15, min: 0, max: 100); // Or truncate any distribution var normal = new Normal(50, 15); @@ -305,7 +784,7 @@ var component1 = new Normal(100, 10); var component2 = new Normal(150, 15); var weights = new double[] { 0.6, 0.4 }; // 60% from first, 40% from second -var mixture = new Mixture(new IUnivariateDistribution[] { component1, component2 }, weights); +var mixture = new Mixture(weights, new UnivariateDistributionBase[] { component1, component2 }); // PDF will show two peaks double pdf = mixture.PDF(125); // Valley between modes @@ -333,7 +812,7 @@ Console.WriteLine($"Empirical 90th percentile: {q90:F2}"); Smooth non-parametric density estimation: ```cs -var kde = new KernelDensity(observations, bandwidth: 1.5); +var kde = new KernelDensity(observations, KernelDensity.KernelType.Gaussian, 1.5); // Smooth PDF double density = kde.PDF(15.0); @@ -352,13 +831,60 @@ For expert judgment and project management: var pert = new Pert(min: 10, mode: 15, max: 25); // PERT from percentile judgments -var pertPercentile = new PertPercentile(p10: 12, p50: 15, p90: 22); +var pertPercentile = new PertPercentile(fifth: 12, fiftieth: 15, ninetyFifth: 22); // Use for duration or cost uncertainty double expectedDuration = pert.Mean; double variance = pert.Variance; ``` +### Competing Risks + +Model the distribution of the minimum (or maximum) of multiple independent or correlated random variables. This is commonly used in system reliability analysis where failure occurs when any component fails: + +```cs +using Numerics.Distributions; +using Numerics.Data.Statistics; + +// Define failure modes for a levee system +var overtopping = new Normal(18.5, 2.0); // Overtopping stage (ft) +var seepage = new LogNormal(2.85, 0.15); // Seepage failure stage (ft) +var erosion = new GeneralizedExtremeValue(16.0, 2.5, -0.1); // Erosion failure stage (ft) + +// System fails at the MINIMUM failure stage +var system = new CompetingRisks(new UnivariateDistributionBase[] { + overtopping, seepage, erosion +}); + +// Default: MinimumOfRandomVariables = true (first failure) +Console.WriteLine("System Failure Analysis (Independent Components):"); +Console.WriteLine($" Mean failure stage: {system.Mean:F2} ft"); +Console.WriteLine($" Median failure stage: {system.InverseCDF(0.5):F2} ft"); +Console.WriteLine($" P(failure ≤ 15 ft): {system.CDF(15.0):F4}"); +Console.WriteLine($" 1% failure stage: {system.InverseCDF(0.01):F2} ft"); +``` + +**Dependency options:** + +```cs +// Independent components (default) +system.Dependency = Probability.DependencyType.Independent; + +// Perfectly correlated — system CDF equals the weakest component +system.Dependency = Probability.DependencyType.PerfectlyPositive; + +// Custom correlation structure +system.Dependency = Probability.DependencyType.CorrelationMatrix; +system.CorrelationMatrix = new double[,] { + { 1.0, 0.6, 0.3 }, + { 0.6, 1.0, 0.4 }, + { 0.3, 0.4, 1.0 } +}; + +// Switch to maximum of random variables +system.MinimumOfRandomVariables = false; +``` + ## Random Number Generation All distributions can generate random samples: @@ -407,7 +933,7 @@ foreach (var T in returnPeriods) ### Example 2: Probability of Exceedance ```cs -var lp3 = new LogPearsonTypeIII(mu: 10.2, sigma: 0.3, gamma: 0.4); +var lp3 = new LogPearsonTypeIII(meanOfLog: 10.2, standardDeviationOfLog: 0.3, skewOfLog: 0.4); // What's the probability a flood exceeds 50,000 cfs? double threshold = 50000; @@ -450,7 +976,7 @@ foreach (var p in probs) ```cs // Component with Weibull failure time distribution -var weibull = new Weibull(alpha: 1000, beta: 2.5); // hours +var weibull = new Weibull(scale: 1000, shape: 2.5); // hours // Reliability at time t (probability of survival) double t = 500; // hours @@ -470,7 +996,7 @@ Console.WriteLine($" MTTF: {weibull.Mean:F1} hours"); ```cs // Annual probability of dam failure -var failureProb = new Beta(alpha: 2, beta: 1998); // ~0.001 +var failureProb = new BetaDistribution(alpha: 2, beta: 1998); // ~0.001 // Generate scenarios double[] scenarios = failureProb.GenerateRandomValues(10000, seed: 12345); @@ -498,6 +1024,7 @@ Console.WriteLine($"Scenarios > 0.002: {scenarios.Count(x => x > 0.002)} / 10000 | Count data | Poisson, Binomial | | Expert judgment | PERT, PERT Percentile, Triangular | | Non-parametric | Empirical, Kernel Density | +| Circular/directional data | Von Mises | ## Parameter Bounds and Validation @@ -507,12 +1034,12 @@ All distributions validate parameters: var gev = new GeneralizedExtremeValue(); // Check if parameters are valid -var params = new double[] { 1000, 200, 0.3 }; -var exception = gev.ValidateParameters(params, throwException: false); +var parameters = new double[] { 1000, 200, 0.3 }; +var exception = gev.ValidateParameters(parameters, throwException: false); if (exception == null) { - gev.SetParameters(params); + gev.SetParameters(parameters); Console.WriteLine("Parameters are valid"); } else @@ -553,16 +1080,96 @@ for (int i = 0; i < normal.NumberOfParameters; i++) } ``` +## Parameter Interpretation Guide + +Most continuous distributions can be understood through three fundamental types of parameters, each controlling a distinct aspect of the distribution's behavior: + +### Location Parameters ($\xi$, $\mu$) + +Location parameters shift the entire distribution left or right along the real line without changing its shape or spread. Changing the location parameter translates every quantile by the same amount. + +- **Normal:** $\mu$ (mean) shifts the center of symmetry +- **Gumbel / GEV:** $\xi$ shifts the mode +- **Exponential:** $\xi$ shifts the lower bound of support + +### Scale Parameters ($\alpha$, $\sigma$, $\theta$) + +Scale parameters stretch or compress the distribution. Multiplying the scale by a constant $c$ multiplies the standard deviation (and all quantile deviations from the location) by $c$, without changing the shape. + +- **Normal:** $\sigma$ (standard deviation) controls the spread +- **Gumbel / GEV / Exponential:** $\alpha$ controls the spread +- **Gamma:** $\theta$ scales the distribution; mean = $\kappa\theta$ + +### Shape Parameters ($\kappa$, $\alpha$, $\beta$, $\gamma$) + +Shape parameters control the fundamental character of the distribution: tail weight, skewness, peakedness, and boundedness. Unlike location and scale, changing a shape parameter alters the qualitative behavior of the distribution. + +- **GEV:** $\kappa$ determines the tail type (bounded vs. heavy-tailed vs. exponential) +- **Gamma:** $\kappa$ controls skewness ($2/\sqrt{\kappa}$); as $\kappa \to \infty$, the Gamma approaches the Normal +- **Beta:** $\alpha$ and $\beta$ jointly control the shape on $[0,1]$; equal values produce symmetry +- **LP3:** $\gamma$ (skewness of log) controls asymmetry; $\gamma = 0$ reduces LP3 to Log-Normal + +## Distribution Relationships + +Many distributions in the ***Numerics*** library are connected through limiting cases, transformations, or special parameterizations. Understanding these relationships helps in selecting appropriate models and verifying analytical results. + +### Extreme Value Family + +``` +GEV(ξ, α, κ) +├── κ = 0 → Gumbel(ξ, α) [Type I: exponential tails] +├── κ > 0 → Weibull-type [Type III: bounded upper tail] +└── κ < 0 → Fréchet-type [Type II: heavy upper tail] +``` + +The Gumbel distribution `Gumbel(ξ, α)` is exactly equivalent to `GeneralizedExtremeValue(ξ, α, 0)`. + +### Gamma Family + +``` +GammaDistribution(θ, κ) +├── κ = 1 → Exponential(0, θ) [memoryless special case] +├── κ = ν/2, θ=2 → Chi-Squared(ν) [sum of squared standard Normals] +└── κ → ∞ → Normal(κθ, θ√κ) [by Central Limit Theorem] +``` + +### Logarithmic Transforms + +``` +X ~ LogNormal(μ, σ) ⟺ log(X) ~ Normal(μ, σ) +X ~ LogPearsonTypeIII(μ, σ, γ) ⟺ log(X) ~ PearsonTypeIII(μ, σ, γ) +``` + +When the LP3 skewness parameter $\gamma = 0$, the LP3 reduces to the Log-Normal distribution. + +### Beta and Uniform + +``` +Beta(1, 1) = Uniform(0, 1) +``` + +The Beta distribution with both shape parameters equal to 1 produces a uniform density on $[0, 1]$. + +### Student's t and Normal + +``` +Student-t(ν) → Normal(0, 1) as ν → ∞ +``` + +The Student's t distribution approaches the standard Normal distribution as the degrees of freedom increase. + --- ## References -[1] Bulletin 17C: Guidelines for Determining Flood Flow Frequency. (2017). U.S. Geological Survey Techniques and Methods, Book 4, Chapter B5. +[1] Interagency Advisory Committee on Water Data. (2019). *Guidelines for Determining Flood Flow Frequency, Bulletin 17C*. U.S. Geological Survey Techniques and Methods, Book 4, Chapter B5. [2] Coles, S. (2001). *An Introduction to Statistical Modeling of Extreme Values*. Springer. [3] Hosking, J. R. M., & Wallis, J. R. (1997). *Regional Frequency Analysis: An Approach Based on L-Moments*. Cambridge University Press. +[4] Johnson, N. L., Kotz, S., & Balakrishnan, N. (1994-1995). *Continuous Univariate Distributions*, Vols. 1-2 (2nd ed.). Wiley. + --- -[← Back to Index](../index.md) | [Next: Parameter Estimation →](parameter-estimation.md) +[← Previous: Hypothesis Tests](../statistics/hypothesis-tests.md) | [Back to Index](../index.md) | [Next: Parameter Estimation →](parameter-estimation.md) diff --git a/docs/example-data/README.md b/docs/example-data/README.md new file mode 100644 index 00000000..06874f9f --- /dev/null +++ b/docs/example-data/README.md @@ -0,0 +1,38 @@ +# Example Data + +This folder contains datasets used in the ***Numerics*** library documentation tutorials. Each dataset is sourced from published references and validated against established statistical software (R, rstan, HEC-SSP). + +## Datasets + +| File | Description | Records | Source | +|------|-------------|---------|--------| +| `tippecanoe-river-streamflow.csv` | Annual peak streamflow, Tippecanoe River near Delphi, IN (Station 43) | 48 | Rao & Hamed (2000), Table 5.1.1 | +| `white-river-nora-floods.csv` | Annual peak streamflow, White River near Nora, IN | 62 | Rao & Hamed (2000), Table 7.1.2 | +| `white-river-mt-carmel-exceedances.csv` | Threshold exceedances (≥50,000 cfs), White River at Mt. Carmel, IN | 281 | Rao & Hamed (2000), Table 8.3.1 | +| `usgs-01562000-streamflow.csv` | Annual peak streamflow with empirical probabilities, USGS 01562000 | 99 | USGS Bulletin 17C test sites | +| `iris-dataset.csv` | Fisher's Iris flower measurements, 3 species | 150 | Fisher (1936) | +| `statistics-samples.csv` | Two general-purpose samples for statistical analysis | 69 | Validated against R (2024) | +| `streamflow-regression.csv` | Regional watershed characteristics and annual peak flows | 25 | Synthetic data modeled after USGS regional regression relationships | +| `flood-damage-glm.csv` | Flood hydraulics and binary damage outcomes | 35 | Synthetic data for logistic regression modeling | + +## References + +- Rao, A. R., & Hamed, K. H. (2000). *Flood Frequency Analysis*. CRC Press. +- Fisher, R. A. (1936). The use of multiple measurements in taxonomic problems. *Annals of Eugenics*, 7(2), 179-188. +- U.S. Geological Survey. (2018). Guidelines for Determining Flood Flow Frequency — Bulletin 17C. +- R Core Team (2024). R: A Language and Environment for Statistical Computing. + +## Loading Data in C# + +```cs +// Example: Load CSV data for analysis +using System.IO; +using System.Linq; + +string[] lines = File.ReadAllLines("example-data/tippecanoe-river-streamflow.csv"); +double[] data = lines + .Where(line => !line.StartsWith("#") && !string.IsNullOrWhiteSpace(line)) + .Skip(1) // Skip header + .Select(line => double.Parse(line.Trim())) + .ToArray(); +``` diff --git a/docs/example-data/flood-damage-glm.csv b/docs/example-data/flood-damage-glm.csv new file mode 100644 index 00000000..232036cd --- /dev/null +++ b/docs/example-data/flood-damage-glm.csv @@ -0,0 +1,39 @@ +# Flood Damage Binary Outcome Data +# Synthetic data for logistic regression modeling +# DamageOccurred: 1 = structure damage observed, 0 = no damage +FloodStage_ft,Duration_hr,Velocity_fps,DamageOccurred +8.2,4,1.5,0 +12.5,8,3.2,0 +15.0,12,4.1,1 +10.3,6,2.0,0 +18.5,24,5.8,1 +7.1,3,1.2,0 +14.2,10,3.8,1 +9.5,5,1.8,0 +16.8,18,5.0,1 +11.0,7,2.5,0 +13.5,9,3.5,0 +19.2,30,6.2,1 +8.8,4,1.6,0 +17.5,22,5.5,1 +10.8,6,2.3,0 +15.5,14,4.5,1 +12.0,8,2.8,0 +20.0,36,6.8,1 +9.0,4,1.7,0 +14.8,11,4.0,1 +11.5,7,2.6,0 +16.2,16,4.8,1 +13.0,8,3.2,0 +18.0,26,5.6,1 +7.5,3,1.3,0 +15.8,15,4.6,1 +10.0,5,2.1,0 +17.0,20,5.2,1 +12.8,9,3.0,0 +19.5,32,6.5,1 +8.5,4,1.5,0 +14.5,10,3.9,1 +11.2,7,2.4,0 +16.5,17,4.9,1 +13.8,9,3.6,1 diff --git a/docs/example-data/iris-dataset.csv b/docs/example-data/iris-dataset.csv new file mode 100644 index 00000000..5fde90ad --- /dev/null +++ b/docs/example-data/iris-dataset.csv @@ -0,0 +1,154 @@ +# Fisher's Iris Dataset +# 150 samples, 3 species (1=Setosa, 2=Versicolor, 3=Virginica) +# Source: R.A. Fisher (1936), "The use of multiple measurements in taxonomic problems" +SepalLength,SepalWidth,PetalLength,PetalWidth,Species +5.1,3.5,1.4,0.2,1 +4.9,3.0,1.4,0.2,1 +4.7,3.2,1.3,0.2,1 +4.6,3.1,1.5,0.2,1 +5.0,3.6,1.4,0.2,1 +5.4,3.9,1.7,0.4,1 +4.6,3.4,1.4,0.3,1 +5.0,3.4,1.5,0.2,1 +4.4,2.9,1.4,0.2,1 +4.9,3.1,1.5,0.1,1 +5.4,3.7,1.5,0.2,1 +4.8,3.4,1.6,0.2,1 +4.8,3.0,1.4,0.1,1 +4.3,3.0,1.1,0.1,1 +5.8,4.0,1.2,0.2,1 +5.7,4.4,1.5,0.4,1 +5.4,3.9,1.3,0.4,1 +5.1,3.5,1.4,0.3,1 +5.7,3.8,1.7,0.3,1 +5.1,3.8,1.5,0.3,1 +5.4,3.4,1.7,0.2,1 +5.1,3.7,1.5,0.4,1 +4.6,3.6,1.0,0.2,1 +5.1,3.3,1.7,0.5,1 +4.8,3.4,1.9,0.2,1 +5.0,3.0,1.6,0.2,1 +5.0,3.4,1.6,0.4,1 +5.2,3.5,1.5,0.2,1 +5.2,3.4,1.4,0.2,1 +4.7,3.2,1.6,0.2,1 +4.8,3.1,1.6,0.2,1 +5.4,3.4,1.5,0.4,1 +5.2,4.1,1.5,0.1,1 +5.5,4.2,1.4,0.2,1 +4.9,3.1,1.5,0.2,1 +5.0,3.2,1.2,0.2,1 +5.5,3.5,1.3,0.2,1 +4.9,3.6,1.4,0.1,1 +4.4,3.0,1.3,0.2,1 +5.1,3.4,1.5,0.2,1 +5.0,3.5,1.3,0.3,1 +4.5,2.3,1.3,0.3,1 +4.4,3.2,1.3,0.2,1 +5.0,3.5,1.6,0.6,1 +5.1,3.8,1.9,0.4,1 +4.8,3.0,1.4,0.3,1 +5.1,3.8,1.6,0.2,1 +5.3,3.7,1.5,0.2,1 +5.0,3.3,1.4,0.2,1 +5.0,3.0,1.6,0.2,1 +7.0,3.2,4.7,1.4,2 +6.4,3.2,4.5,1.5,2 +6.9,3.1,4.9,1.5,2 +5.5,2.3,4.0,1.3,2 +6.5,2.8,4.6,1.5,2 +5.7,2.8,4.5,1.3,2 +6.3,3.3,4.7,1.6,2 +4.9,2.4,3.3,1.0,2 +6.6,2.9,4.6,1.3,2 +5.2,2.7,3.9,1.4,2 +5.0,2.0,3.5,1.0,2 +5.9,3.0,4.2,1.5,2 +6.0,2.2,4.0,1.0,2 +6.1,2.9,4.7,1.4,2 +5.6,2.9,3.6,1.3,2 +6.7,3.1,4.4,1.4,2 +5.6,3.0,4.5,1.5,2 +5.8,2.7,4.1,1.0,2 +6.2,2.2,4.5,1.5,2 +5.6,2.5,3.9,1.1,2 +5.9,3.2,4.8,1.8,2 +6.1,2.8,4.0,1.3,2 +6.3,2.5,4.9,1.5,2 +6.1,2.8,4.7,1.2,2 +6.4,2.9,4.3,1.3,2 +6.6,3.0,4.4,1.4,2 +6.8,2.8,4.8,1.4,2 +6.7,3.0,5.0,1.7,2 +6.0,2.9,4.5,1.5,2 +5.7,2.6,3.5,1.0,2 +5.5,2.4,3.8,1.1,2 +5.5,2.4,3.7,1.0,2 +5.8,2.7,3.9,1.2,2 +6.0,2.7,5.1,1.6,2 +5.4,3.0,4.5,1.5,2 +6.0,3.4,4.5,1.6,2 +6.7,3.1,4.7,1.5,2 +6.3,2.3,4.4,1.3,2 +5.6,3.0,4.1,1.3,2 +5.5,2.5,4.0,1.3,2 +5.5,2.6,4.4,1.2,2 +6.1,3.0,4.6,1.4,2 +5.8,2.6,4.0,1.2,2 +5.0,2.3,3.3,1.0,2 +5.6,2.7,4.2,1.3,2 +5.7,3.0,4.2,1.2,2 +5.7,2.9,4.2,1.3,2 +6.2,2.9,4.3,1.3,2 +5.1,2.5,3.0,1.1,2 +5.7,2.8,4.1,1.3,2 +6.3,3.3,6.0,2.5,3 +5.8,2.7,5.1,1.9,3 +7.1,3.0,5.9,2.1,3 +6.3,2.9,5.6,1.8,3 +6.5,3.0,5.8,2.2,3 +7.6,3.0,6.6,2.1,3 +4.9,2.5,4.5,1.7,3 +7.3,2.9,6.3,1.8,3 +6.7,2.5,5.8,1.8,3 +7.2,3.6,6.1,2.5,3 +6.5,3.2,5.1,2.0,3 +6.4,2.7,5.3,1.9,3 +6.8,3.0,5.5,2.1,3 +5.7,2.5,5.0,2.0,3 +5.8,2.8,5.1,2.4,3 +6.4,3.2,5.3,2.3,3 +6.5,3.0,5.5,1.8,3 +7.7,3.8,6.7,2.2,3 +7.7,2.6,6.9,2.3,3 +6.0,2.2,5.0,1.5,3 +6.9,3.2,5.7,2.3,3 +5.6,2.8,4.9,2.0,3 +7.7,2.8,6.7,2.0,3 +6.3,2.7,4.9,1.8,3 +6.7,3.3,5.7,2.1,3 +7.2,3.2,6.0,1.8,3 +6.2,2.8,4.8,1.8,3 +6.1,3.0,4.9,1.8,3 +6.4,2.8,5.6,2.1,3 +7.2,3.0,5.8,1.6,3 +7.4,2.8,6.1,1.9,3 +7.9,3.8,6.4,2.0,3 +6.4,2.8,5.6,2.2,3 +6.3,2.8,5.1,1.5,3 +6.1,2.6,5.6,1.4,3 +7.7,3.0,6.1,2.3,3 +6.3,3.4,5.6,2.4,3 +6.4,3.1,5.5,1.8,3 +6.0,3.0,4.8,1.8,3 +6.9,3.1,5.4,2.1,3 +6.7,3.1,5.6,2.4,3 +6.9,3.1,5.1,2.3,3 +5.8,2.7,5.1,1.9,3 +6.8,3.2,5.9,2.3,3 +6.7,3.3,5.7,2.5,3 +6.7,3.0,5.2,2.3,3 +6.3,2.5,5.0,1.9,3 +6.5,3.0,5.2,2.0,3 +6.2,3.4,5.4,2.3,3 +5.9,3.0,5.1,1.8,3 diff --git a/docs/example-data/statistics-samples.csv b/docs/example-data/statistics-samples.csv new file mode 100644 index 00000000..db4178bd --- /dev/null +++ b/docs/example-data/statistics-samples.csv @@ -0,0 +1,73 @@ +# General Statistical Test Samples +# Two independent samples for descriptive statistics, hypothesis tests, and L-moments +# Source: Validated against R Core Team (2024) and R packages (psych, EnvStats, lmom) +Sample1,Sample2 +122,279 +244,105 +214,171 +173,171 +229,129 +156,127 +212,194 +263,234 +146,251 +183,152 +161,207 +205,205 +135,183 +331,137 +225,148 +174,189 +98.8,182 +149,236 +238,148 +262,150 +132,207 +235,252 +216,237 +240,209 +230,225 +192,137 +195,207 +172,129 +173,148 +172,192 +153,95 +142,231 +317,255 +161,220 +201,205 +204,163 +194,265 +164,190 +183,226 +161,123 +167,108 +179,145 +185,197 +117,233 +192,133 +337,177 +125,211 +166,180 +99.1,200 +202,197 +230,142 +158,166 +262,251 +154,254 +164,226 +182,197 +164,250 +183,194 +171,190 +250,181 +184,290 +205,185 +237,123 +177,208 +239,238 +187,179 +180,189 +173,225 +174,236 diff --git a/docs/example-data/streamflow-regression.csv b/docs/example-data/streamflow-regression.csv new file mode 100644 index 00000000..a54474da --- /dev/null +++ b/docs/example-data/streamflow-regression.csv @@ -0,0 +1,29 @@ +# Regional Streamflow Regression Data +# Synthetic data modeled after USGS regional regression relationships +# for watersheds in the central United States +DrainageArea_sqmi,MeanAnnualPrecip_in,MeanElevation_ft,AnnualPeakFlow_cfs +12.5,38.2,820,1450 +25.0,40.1,790,2680 +48.3,39.5,850,4520 +75.0,41.0,780,6100 +102.0,42.3,760,7850 +18.7,37.8,830,2050 +55.0,40.5,810,5200 +130.0,43.0,740,9300 +8.2,36.5,870,980 +200.0,44.2,720,12500 +35.0,39.0,840,3400 +88.0,41.5,770,6900 +150.0,43.5,730,10200 +42.0,39.8,825,4100 +65.0,40.8,800,5650 +110.0,42.8,750,8500 +22.0,38.5,835,2350 +170.0,43.8,725,11200 +95.0,42.0,765,7400 +30.0,38.8,845,3050 +145.0,43.2,735,9800 +60.0,40.2,805,5400 +180.0,44.0,718,11800 +15.0,37.5,855,1700 +80.0,41.2,775,6500 diff --git a/docs/example-data/tippecanoe-river-streamflow.csv b/docs/example-data/tippecanoe-river-streamflow.csv new file mode 100644 index 00000000..5cbfb852 --- /dev/null +++ b/docs/example-data/tippecanoe-river-streamflow.csv @@ -0,0 +1,52 @@ +# Tippecanoe River Near Delphi, Indiana (Station 43) +# Annual Peak Streamflow (cfs) +# Source: "Flood Frequency Analysis", A.R. Rao & K.H. Hamed, CRC Press, 2000, Table 5.1.1 +AnnualPeakFlow +6290 +2700 +13100 +16900 +14600 +9600 +7740 +8490 +8130 +12000 +17200 +15000 +12400 +6960 +6500 +5840 +10400 +18800 +21400 +22600 +14200 +11000 +12800 +15700 +4740 +6950 +11800 +12100 +20600 +14600 +14600 +8900 +10600 +14200 +14100 +14100 +12500 +7530 +13400 +17600 +13400 +19200 +16900 +15500 +14500 +21900 +10400 +7460 diff --git a/docs/example-data/usgs-01562000-streamflow.csv b/docs/example-data/usgs-01562000-streamflow.csv new file mode 100644 index 00000000..602f7efa --- /dev/null +++ b/docs/example-data/usgs-01562000-streamflow.csv @@ -0,0 +1,103 @@ +# USGS 01562000 - Bulletin 17C Test Site +# Annual Peak Streamflow with Empirical Probabilities +# Source: USGS Bulletin 17C test sites, validated against Palisade @Risk +Flow,Probability +3180,0.010036801605888 +4340,0.020073603211777 +4670,0.030110404817665 +4720,0.040147206423553 +5020,0.050184008029441 +6180,0.060220809635329 +6270,0.070257611241218 +7410,0.080294412847106 +7800,0.090331214452994 +8130,0.100368016058882 +8320,0.110404817664771 +8400,0.120441619270659 +8450,0.130478420876547 +8640,0.140515222482436 +8690,0.150552024088324 +8900,0.160588825694212 +8990,0.170625627300100 +9040,0.180662428905989 +9220,0.190699230511877 +9640,0.200736032117765 +9830,0.210772833723653 +10200,0.220809635329542 +10300,0.230846436935430 +10600,0.240883238541318 +10800,0.250920040147206 +10800,0.260956841753095 +11100,0.270993643358983 +11100,0.281030444964871 +11300,0.291067246570759 +11600,0.301104048176648 +11700,0.311140849782536 +11700,0.321177651388424 +11800,0.331214452994312 +11800,0.341251254600201 +12000,0.351288056206089 +12200,0.361324857811977 +12200,0.371361659417865 +12300,0.381398461023754 +12500,0.391435262629642 +12600,0.401472064235530 +12700,0.411508865841419 +12700,0.421545667447307 +12900,0.431582469053195 +13200,0.441619270659083 +13200,0.451656072264972 +13400,0.461692874870860 +13400,0.471729675476748 +13600,0.481766477082636 +13800,0.491803278688525 +14000,0.501840080294413 +14100,0.511876881900301 +14500,0.521913683506189 +14500,0.531950485112078 +14600,0.541987286717966 +15100,0.552024088323854 +15100,0.562060889929742 +15200,0.572097691535631 +15600,0.582134493141519 +16200,0.592171294747407 +17200,0.602208096353295 +17400,0.612244897959184 +17700,0.622281699565072 +17700,0.632318501170960 +17800,0.642355302776848 +18000,0.652392104382737 +18300,0.662428905988625 +18400,0.672465707594513 +18400,0.682502509200401 +18400,0.692539310806290 +18500,0.702576112412178 +18500,0.712612914018066 +18600,0.722649715623955 +18900,0.732686517229843 +19100,0.742723318835731 +19200,0.752760120441619 +19400,0.762796922047508 +19900,0.772833723653396 +20400,0.782870525259284 +20900,0.792907326865172 +21000,0.802944128471061 +21200,0.812980930076949 +21500,0.823017731682837 +21800,0.833054533288725 +22100,0.843091334894614 +22300,0.853128136500502 +22400,0.863164937106390 +22500,0.873201739712278 +22700,0.883238541318167 +22800,0.893275342924055 +23600,0.903312144529943 +26800,0.913348946135831 +29000,0.923385747741720 +31300,0.933422549347608 +39200,0.943459350953496 +40200,0.953496152559384 +42900,0.963532954165273 +45800,0.973569755771161 +71300,0.989071038251366 +80500,0.994535519125683 diff --git a/docs/example-data/white-river-mt-carmel-exceedances.csv b/docs/example-data/white-river-mt-carmel-exceedances.csv new file mode 100644 index 00000000..336b06ab --- /dev/null +++ b/docs/example-data/white-river-mt-carmel-exceedances.csv @@ -0,0 +1,285 @@ +# White River at Mt. Carmel, Indiana +# Threshold Exceedances (cfs), Threshold = 50,000 cfs +# Source: "Flood Frequency Analysis", A.R. Rao & K.H. Hamed, CRC Press, 2000, Table 8.3.1 +PeakFlow +126000 +148000 +66000 +156000 +136000 +122000 +183000 +162000 +85200 +56800 +56600 +138000 +81000 +51800 +90700 +139000 +160000 +118000 +50600 +137000 +151000 +172000 +52600 +248000 +152000 +64500 +61500 +143000 +108000 +53000 +134000 +115000 +84100 +105000 +85400 +76900 +99100 +73700 +122000 +62500 +54300 +58000 +144000 +55800 +127000 +55800 +107000 +56400 +128000 +106000 +110000 +232000 +60400 +60400 +50800 +100000 +55900 +167000 +53700 +56700 +126000 +100000 +59500 +164000 +81800 +56400 +124000 +64600 +77300 +65900 +72500 +65100 +80800 +69800 +53000 +195000 +128000 +114000 +110000 +149000 +74100 +75900 +99300 +168000 +70800 +104000 +125000 +77300 +97300 +140000 +54900 +66000 +199000 +99800 +105000 +93700 +277000 +85700 +77300 +122000 +106000 +93300 +79000 +130000 +126000 +57800 +64700 +162000 +71900 +63500 +81500 +51000 +84400 +108000 +185000 +55800 +94600 +82800 +146000 +66500 +57700 +78700 +85100 +129000 +75700 +104000 +139000 +50600 +53500 +178000 +110000 +50800 +76000 +130000 +67300 +149000 +78400 +96600 +83300 +68400 +84300 +56400 +112000 +76400 +116000 +51400 +59800 +63900 +81900 +88200 +62300 +162000 +67200 +85500 +51000 +286000 +73800 +61300 +60800 +91300 +134000 +106000 +70800 +106000 +122000 +149000 +53700 +85300 +144000 +54800 +116000 +67500 +56500 +86700 +91500 +105000 +134000 +97300 +84000 +141000 +52600 +124000 +196000 +84200 +54500 +74500 +104000 +57200 +61000 +155000 +96500 +89100 +77900 +70500 +73400 +180000 +83700 +302000 +133000 +92100 +105000 +235000 +213000 +96100 +77100 +73900 +55400 +55200 +87800 +52600 +106000 +93000 +147000 +61800 +101000 +154000 +52000 +121000 +86700 +57300 +97500 +112000 +88500 +76200 +140000 +87400 +154000 +95100 +131000 +131000 +54900 +78800 +101000 +224000 +54800 +50900 +63500 +63500 +152000 +51000 +285000 +114000 +197000 +106000 +132000 +83700 +67200 +110000 +202000 +127000 +90600 +126000 +73900 +86500 +181000 +141000 +79700 +97800 +57300 +77200 +133000 +82900 +55000 +62000 +51700 +54500 +51600 +103000 +134000 +71700 +57000 +63900 +60700 +81900 +171000 +111000 +50400 +50500 +69700 +88900 +76600 diff --git a/docs/example-data/white-river-nora-floods.csv b/docs/example-data/white-river-nora-floods.csv new file mode 100644 index 00000000..1cbc33a2 --- /dev/null +++ b/docs/example-data/white-river-nora-floods.csv @@ -0,0 +1,66 @@ +# White River near Nora, Indiana +# Annual Peak Streamflow (cfs) +# Source: "Flood Frequency Analysis", A.R. Rao & K.H. Hamed, CRC Press, 2000, Table 7.1.2 +AnnualPeakFlow +23200 +2950 +10300 +23200 +4540 +9960 +10800 +26900 +23300 +20400 +8480 +3150 +9380 +32400 +20800 +11100 +7270 +9600 +14600 +14300 +22500 +14700 +12700 +9740 +3050 +8830 +12000 +30400 +27000 +15200 +8040 +11700 +20300 +22700 +30400 +9180 +4870 +14700 +12800 +13700 +7960 +9830 +12500 +10700 +13200 +14700 +14300 +4050 +14600 +14400 +19200 +7160 +12100 +8650 +10600 +24500 +14400 +6300 +9560 +15800 +14300 +28700 diff --git a/docs/getting-started.md b/docs/getting-started.md index 91a9a551..b363c80d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,5 +1,7 @@ # Getting Started +[Back to Index](index.md) | [Next: Numerical Integration →](mathematics/integration.md) + This guide will help you get up and running with the ***Numerics*** library quickly. ## Installation @@ -9,13 +11,13 @@ This guide will help you get up and running with the ***Numerics*** library quic The easiest way to install ***Numerics*** is via NuGet: ```bash -dotnet add package Numerics +dotnet add package RMC.Numerics ``` -Or add the following to your `.csproj` file: +Or using the Package Manager Console: -```xml - +``` +Install-Package RMC.Numerics ``` ### Manual Installation @@ -24,7 +26,7 @@ Download the compiled DLL from the releases page and add a reference to your pro ```xml - path\to\Numerics.dll + Numerics.dll ``` @@ -41,7 +43,6 @@ using Numerics.Data.Statistics; // Numerical methods using Numerics.Mathematics.Integration; -using Numerics.Mathematics.Differentiation; using Numerics.Mathematics.Optimization; using Numerics.Mathematics.LinearAlgebra; using Numerics.Mathematics.RootFinding; @@ -51,7 +52,7 @@ using Numerics.Sampling; using Numerics.Sampling.MCMC; // Interpolation -using Numerics.Data.Interpolation; +using Numerics.Data; ``` ## Working with Distributions @@ -105,7 +106,7 @@ Console.WriteLine($"Median: {dist.Median}"); Console.WriteLine($"Mode: {dist.Mode}"); Console.WriteLine($"Variance: {dist.Variance}"); Console.WriteLine($"Std Dev: {dist.StandardDeviation}"); -Console.WriteLine($"Skewness: {dist.Skew}"); +Console.WriteLine($"Skewness: {dist.Skewness}"); Console.WriteLine($"Kurtosis: {dist.Kurtosis}"); ``` @@ -120,66 +121,67 @@ var dist = new Normal(100, 15); double x = dist.InverseCDF(new Random().NextDouble()); // Multiple random values (more efficient) -double[] samples = new double[1000]; -dist.GenerateRandomValues(samples); +double[] samples = dist.GenerateRandomValues(1000); // With specific seed for reproducibility -dist.GenerateRandomValues(samples, seed: 12345); +double[] seededSamples = dist.GenerateRandomValues(1000, seed: 12345); ``` ## Fitting Distributions to Data ### Parameter Estimation Methods -***Numerics*** supports three estimation methods: +***Numerics*** supports multiple estimation methods. The simplest approach is `Estimate()`: ```cs -double[] data = { 10.2, 15.1, 12.3, 18.7, 14.2, 16.8, 13.1, 17.5 }; +using Numerics.Distributions; -var normal = new Normal(); +double[] data = { 10.2, 15.1, 12.3, 18.7, 14.2, 16.8, 13.1, 17.5 }; -// Method of Moments (MOM) -if (normal is IMomentEstimation mom) -{ - normal.SetParameters(mom.ParametersFromMoments(data)); -} +// L-Moments (recommended for hydrological data) +var gev = new GeneralizedExtremeValue(); +gev.Estimate(data, ParameterEstimationMethod.MethodOfLinearMoments); -// L-Moments (LMOM) - preferred for heavy-tailed distributions -if (normal is ILinearMomentEstimation lmom) -{ - normal.SetParameters(lmom.ParametersFromLinearMoments(data)); -} +// Maximum Likelihood Estimation +var normal = new Normal(); +normal.Estimate(data, ParameterEstimationMethod.MaximumLikelihood); -// Maximum Likelihood Estimation (MLE) -if (normal is IMaximumLikelihoodEstimation mle) -{ - normal.SetParameters(mle.MLE(data)); -} +// Method of Moments +var lognormal = new LogNormal(); +lognormal.Estimate(data, ParameterEstimationMethod.MethodOfMoments); ``` ### Hydrologic Frequency Analysis Example +This example uses annual peak streamflow from the Tippecanoe River near Delphi, Indiana (Rao & Hamed, 2000, Table 5.1.1). See [`example-data/tippecanoe-river-streamflow.csv`](example-data/tippecanoe-river-streamflow.csv). + ```cs using Numerics.Distributions; using Numerics.Data.Statistics; -// Annual maximum streamflow data (cfs) -double[] annualMax = { 12500, 15200, 11800, 18900, 14200, - 16500, 13400, 17800, 10900, 19500 }; +// Tippecanoe River near Delphi, IN — 48 years of annual peak streamflow (cfs) +// Source: Rao & Hamed (2000), Table 5.1.1 +double[] annualPeaks = { + 6290, 2700, 13100, 16900, 14600, 9600, 7740, 8490, 8130, 12000, + 17200, 15000, 12400, 6960, 6500, 5840, 10400, 18800, 21400, 22600, + 14200, 11000, 12800, 15700, 4740, 6950, 11800, 12100, 20600, 14600, + 14600, 8900, 10600, 14200, 14100, 14100, 12500, 7530, 13400, 17600, + 13400, 19200, 16900, 15500, 14500, 21900, 10400, 7460 +}; -// Fit Log-Pearson Type III using L-Moments +// Fit Log-Pearson Type III using L-Moments (USGS Bulletin 17C method) var lp3 = new LogPearsonTypeIII(); -lp3.SetParameters(lp3.ParametersFromLinearMoments(annualMax)); +lp3.Estimate(annualPeaks, ParameterEstimationMethod.MethodOfLinearMoments); -// Compute flood quantiles +// Compute flood quantiles for key return periods Console.WriteLine("Return Period Analysis:"); Console.WriteLine($" 10-year flood (10% AEP): {lp3.InverseCDF(0.90):N0} cfs"); Console.WriteLine($" 50-year flood (2% AEP): {lp3.InverseCDF(0.98):N0} cfs"); Console.WriteLine($" 100-year flood (1% AEP): {lp3.InverseCDF(0.99):N0} cfs"); -Console.WriteLine($" 500-year flood (0.2% AEP):{lp3.InverseCDF(0.998):N0} cfs"); +Console.WriteLine($" 500-year flood (0.2% AEP): {lp3.InverseCDF(0.998):N0} cfs"); // Assess goodness-of-fit -double aic = GoodnessOfFit.AIC(lp3.NumberOfParameters, lp3.LogLikelihood(annualMax)); +double aic = GoodnessOfFit.AIC(lp3.NumberOfParameters, lp3.LogLikelihood(annualPeaks)); Console.WriteLine($"\nAIC: {aic:F2}"); ``` @@ -215,16 +217,17 @@ double[] lower = { 0, 0, 0 }; double[] upper = { 1, 1, 1 }; // Monte Carlo integration -var mc = new MonteCarlo(f3d, lower, upper); -mc.Iterations = 100000; +var mc = new MonteCarloIntegration(f3d, 3, lower, upper); +mc.MaxIterations = 100000; mc.Integrate(); -Console.WriteLine($"Monte Carlo: {mc.Result:F6} ± {mc.Error:F6}"); +Console.WriteLine($"Monte Carlo: {mc.Result:F6} ± {mc.StandardError:F6}"); // VEGAS adaptive importance sampling -var vegas = new Vegas(f3d, lower, upper); -vegas.Iterations = 10000; +Func f3dVegas = (x, w) => x[0] * x[1] * x[2]; +var vegas = new Vegas(f3dVegas, 3, lower, upper); +vegas.MaxIterations = 10000; vegas.Integrate(); -Console.WriteLine($"VEGAS: {vegas.Result:F6} ± {vegas.Error:F6}"); +Console.WriteLine($"VEGAS: {vegas.Result:F6} ± {vegas.StandardError:F6}"); ``` ## Optimization @@ -243,10 +246,10 @@ Func rosenbrock = x => }; // BFGS (quasi-Newton method) -var bfgs = new BFGS(rosenbrock, 2, new double[] { -1, -1 }); +var bfgs = new BFGS(rosenbrock, 2, new double[] { -1, -1 }, new double[] { -5, -5 }, new double[] { 5, 5 }); bfgs.Minimize(); -Console.WriteLine($"BFGS minimum at: ({bfgs.BestParameterSet[0]:F6}, {bfgs.BestParameterSet[1]:F6})"); -Console.WriteLine($"Function value: {bfgs.BestFitness:E6}"); +Console.WriteLine($"BFGS minimum at: ({bfgs.BestParameterSet.Values[0]:F6}, {bfgs.BestParameterSet.Values[1]:F6})"); +Console.WriteLine($"Function value: {bfgs.BestParameterSet.Fitness:E6}"); ``` ### Global Optimization @@ -269,7 +272,7 @@ double[] upper = { 5.12, 5.12 }; // Differential Evolution var de = new DifferentialEvolution(rastrigin, 2, lower, upper); de.Minimize(); -Console.WriteLine($"DE minimum at: ({de.BestParameterSet[0]:F6}, {de.BestParameterSet[1]:F6})"); +Console.WriteLine($"DE minimum at: ({de.BestParameterSet.Values[0]:F6}, {de.BestParameterSet.Values[1]:F6})"); ``` ## MCMC Sampling @@ -291,39 +294,45 @@ var priors = new List }; // Log-likelihood function -double LogLikelihood(double[] theta) +double ComputeLogLikelihood(double[] theta) { double mu = theta[0]; double sigma = theta[1]; if (sigma <= 0) return double.NegativeInfinity; - + var model = new Normal(mu, sigma); return model.LogLikelihood(observations); } // Run DE-MCz sampler -var sampler = new DEMCz(priors, LogLikelihood); +var sampler = new DEMCz(priors, ComputeLogLikelihood); sampler.Iterations = 20000; sampler.WarmupIterations = 5000; sampler.Sample(); -// Analyze results -var output = sampler.Output; -Console.WriteLine($"Mean estimate: {output.Mean(0):F3} [{output.Percentile(0, 0.025):F3}, {output.Percentile(0, 0.975):F3}]"); -Console.WriteLine($"Std estimate: {output.Mean(1):F3} [{output.Percentile(1, 0.025):F3}, {output.Percentile(1, 0.975):F3}]"); +// Analyze results - flatten chains and extract parameter values +var allSamples = sampler.Output.SelectMany(chain => chain).ToList(); + +for (int i = 0; i < 2; i++) +{ + var values = allSamples.Select(ps => ps.Values[i]).ToArray(); + double mean = Statistics.Mean(values); + double lower = Statistics.Percentile(values, 0.025); + double upper = Statistics.Percentile(values, 0.975); + Console.WriteLine($"Parameter {i}: {mean:F3} [{lower:F3}, {upper:F3}]"); +} ``` ## Performance Tips -1. **Use parallelization** for large-scale Monte Carlo simulations: +1. **Use parallelization** for large-scale Monte Carlo simulations (enabled by default; verify it is enabled): ```cs - sampler.ParallelizeChains = true; + sampler.ParallelizeChains = true; // default is true ``` -2. **Pre-allocate arrays** when generating many random values: +2. **Generate random values efficiently**: ```cs - double[] values = new double[100000]; - dist.GenerateRandomValues(values); + double[] values = dist.GenerateRandomValues(100000); ``` 3. **Choose appropriate integration method** based on function smoothness: @@ -345,3 +354,7 @@ Console.WriteLine($"Std estimate: {output.Mean(1):F3} [{output.Percentile(1, 0. ## References [1] Press, W. H., Teukolsky, S. A., Vetterling, W. T., & Flannery, B. P. (2007). *Numerical Recipes: The Art of Scientific Computing* (3rd ed.). Cambridge University Press. + +--- + +[Back to Index](index.md) | [Next: Numerical Integration →](mathematics/integration.md) diff --git a/docs/index.md b/docs/index.md index b6ca652b..21241401 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ The library is designed for engineers, scientists, and researchers who need reli - Multiple parameter estimation methods (Method of Moments, L-Moments, Maximum Likelihood) - Uncertainty analysis via bootstrap resampling - Bivariate copulas for dependency modeling -- Multivariate normal distribution +- Multivariate distributions (Normal, Student-t, Dirichlet, Multinomial) ### Statistical Analysis - Comprehensive goodness-of-fit metrics (NSE, KGE, RMSE, PBIAS, AIC/BIC) @@ -76,7 +76,7 @@ double[] annualMaxFlows = { 1200, 1500, 1100, 1800, 1350, 1600, 1250, 1450 }; // Fit using L-Moments (recommended for hydrologic data) var gev = new GeneralizedExtremeValue(); -gev.SetParameters(gev.ParametersFromLinearMoments(annualMaxFlows)); +gev.Estimate(annualMaxFlows, ParameterEstimationMethod.MethodOfLinearMoments); // Compute the 100-year flood (1% annual exceedance probability) double q100 = gev.InverseCDF(0.99); @@ -109,15 +109,15 @@ var priors = new List new Uniform(0, 100) // Prior for parameter 2 }; -// Define log-likelihood function -double LogLikelihood(double[] parameters) +// Define log-likelihood function (simple Gaussian example) +double ComputeLogLikelihood(double[] parameters) { - // Your likelihood calculation here + // Log-likelihood for parameters[0] with observed value of 5 return -0.5 * Math.Pow(parameters[0] - 5, 2); } // Create and run sampler -var sampler = new DEMCz(priors, LogLikelihood); +var sampler = new DEMCz(priors, ComputeLogLikelihood); sampler.Iterations = 10000; sampler.Sample(); @@ -127,41 +127,38 @@ var results = sampler.Output; ## Documentation Structure -📘 **Status Legend:** -- ✅ = Reviewed and updated with accurate code examples -- 📝 = Draft (needs verification against actual library) - -| Document | Status | Description | -|----------|--------|-------------| -| [Getting Started](getting-started.md) | ✅ | Installation and basic usage patterns | -| **Mathematics** | | | -| [Numerical Integration](mathematics/integration.md) | ✅ | Comprehensive guide to 1D, 2D, and multidimensional integration | -| [Numerical Differentiation](mathematics/differentiation.md) | ✅ | Derivatives, gradients, Hessians, and Jacobians | -| [Optimization](mathematics/optimization.md) | ✅ | Local and global optimization algorithms | -| [Root Finding](mathematics/root-finding.md) | ✅ | Equation solving methods | -| [Linear Algebra](mathematics/linear-algebra.md) | ✅ | Matrix and vector operations | -| [Special Functions](mathematics/special-functions.md) | ✅ | Gamma, Beta, Error functions | -| [ODE Solvers](mathematics/ode-solvers.md) | ✅ | Runge-Kutta methods | -| **Distributions** | | | -| [Univariate Distributions](distributions/univariate.md) | ✅ | Complete reference for univariate distributions | -| [Multivariate Distributions](distributions/multivariate.md) | ✅ | Multivariate Normal distribution | -| [Parameter Estimation](distributions/parameter-estimation.md) | ✅ | Fitting distributions to data | -| [Uncertainty Analysis](distributions/uncertainty-analysis.md) | ✅ | Bootstrap and confidence intervals | -| [Copulas](distributions/copulas.md) | ✅ | Dependency modeling with copulas | -| **Statistics** | | | -| [Descriptive Statistics](statistics/descriptive.md) | ✅ | Summary statistics functions | -| [Goodness-of-Fit](statistics/goodness-of-fit.md) | ✅ | Model evaluation metrics | -| [Hypothesis Tests](statistics/hypothesis-tests.md) | ✅ | Statistical hypothesis testing | -| **Data** | | | -| [Interpolation](data/interpolation.md) | ✅ | Interpolation methods | -| [Time Series](data/time-series.md) | ✅ | Time series data structures and analysis | -| **Machine Learning** | | | -| [Overview](machine-learning/overview.md) | ✅ | Supervised and unsupervised learning algorithms | -| **Sampling** | | | -| [MCMC Methods](sampling/mcmc.md) | ✅ | Markov Chain Monte Carlo samplers | -| [Convergence Diagnostics](sampling/convergence-diagnostics.md) | ✅ | MCMC convergence assessment | -| [Random Generation](sampling/random-generation.md) | ✅ | PRNGs, quasi-random, and sampling methods | -| [References](references.md) | ✅ | Complete bibliography | +| Document | Description | +|----------|-------------| +| [Getting Started](getting-started.md) | Installation and basic usage patterns | +| **Mathematics** | | +| [Numerical Integration](mathematics/integration.md) | Comprehensive guide to 1D, 2D, and multidimensional integration | +| [Numerical Differentiation](mathematics/differentiation.md) | Derivatives, gradients, Hessians, and Jacobians | +| [Optimization](mathematics/optimization.md) | Local and global optimization algorithms | +| [Root Finding](mathematics/root-finding.md) | Equation solving methods | +| [Linear Algebra](mathematics/linear-algebra.md) | Matrix and vector operations, decompositions | +| [Special Functions](mathematics/special-functions.md) | Gamma, Beta, Error functions | +| [ODE Solvers](mathematics/ode-solvers.md) | Runge-Kutta methods for initial value problems | +| **Data** | | +| [Interpolation](data/interpolation.md) | Interpolation methods and splines | +| [Linear Regression](data/regression.md) | Linear regression modeling | +| [Time Series](data/time-series.md) | Time series data structures and analysis | +| **Statistics** | | +| [Descriptive Statistics](statistics/descriptive.md) | Summary statistics and moments | +| [Goodness-of-Fit](statistics/goodness-of-fit.md) | Model evaluation metrics | +| [Hypothesis Tests](statistics/hypothesis-tests.md) | Statistical hypothesis testing | +| **Distributions** | | +| [Univariate Distributions](distributions/univariate.md) | 40+ probability distributions with PDF, CDF, and quantile functions | +| [Parameter Estimation](distributions/parameter-estimation.md) | Fitting distributions to data | +| [Uncertainty Analysis](distributions/uncertainty-analysis.md) | Bootstrap and confidence intervals | +| [Copulas](distributions/copulas.md) | Dependency modeling with copulas | +| [Multivariate Distributions](distributions/multivariate.md) | Multivariate Normal, Student-t, Dirichlet, Multinomial | +| **Machine Learning** | | +| [Machine Learning](machine-learning/machine-learning.md) | Supervised and unsupervised learning algorithms | +| **Sampling** | | +| [Random Generation](sampling/random-generation.md) | PRNGs, quasi-random, and sampling methods | +| [MCMC Methods](sampling/mcmc.md) | Markov Chain Monte Carlo samplers | +| [Convergence Diagnostics](sampling/convergence-diagnostics.md) | MCMC convergence assessment | +| [References](references.md) | Complete bibliography | ## Namespaces @@ -169,15 +166,14 @@ var results = sampler.Output; |-----------|-------------| | `Numerics.Distributions` | Probability distributions and copulas | | `Numerics.Data.Statistics` | Statistical functions and tests | -| `Numerics.Data.Interpolation` | Interpolation methods | -| `Numerics.Data.TimeSeries` | Time series data structures | -| `Numerics.Mathematics` | Base namespace for mathematical operations | +| `Numerics.Data` | Interpolation methods, linear regression, time series data structures | +| `Numerics.Mathematics` | Base namespace for mathematical operations (includes NumericalDerivative) | | `Numerics.Mathematics.Integration` | Numerical integration methods | -| `Numerics.Mathematics.Differentiation` | Numerical differentiation (via NumericalDerivative class) | | `Numerics.Mathematics.Optimization` | Optimization algorithms | | `Numerics.Mathematics.LinearAlgebra` | Matrix and vector operations | | `Numerics.Mathematics.RootFinding` | Root finding algorithms | | `Numerics.Mathematics.SpecialFunctions` | Gamma, Beta, Error functions | +| `Numerics.MachineLearning` | Supervised and unsupervised learning algorithms | | `Numerics.Sampling` | Random sampling and stratification | | `Numerics.Sampling.MCMC` | MCMC samplers and diagnostics | diff --git a/docs/machine-learning/machine-learning.md b/docs/machine-learning/machine-learning.md new file mode 100644 index 00000000..bbd5fe02 --- /dev/null +++ b/docs/machine-learning/machine-learning.md @@ -0,0 +1,1032 @@ +# Machine Learning + +[← Previous: Multivariate Distributions](../distributions/multivariate.md) | [Back to Index](../index.md) | [Next: Random Generation →](../sampling/random-generation.md) + +The ***Numerics*** library provides machine learning algorithms for both supervised and unsupervised learning tasks. These implementations are designed for engineering and scientific applications including classification, regression, and clustering. + +## Overview + +**Supervised Learning:** +- Generalized Linear Models (GLM) +- Decision Trees +- Random Forests +- k-Nearest Neighbors (KNN) +- Naive Bayes + +**Unsupervised Learning:** +- k-Means Clustering +- Gaussian Mixture Models (GMM) +- Jenks Natural Breaks + +--- + +## Supervised Learning + +### Generalized Linear Models (GLM) + +GLMs extend linear regression to non-normal response distributions [[1]](#1). A GLM relates the expected value of the response variable $\mu = E(y)$ to a linear predictor $\eta = X\beta$ through a link function $g$: + +```math +g(\mu) = \eta = X\beta = \beta_0 + \beta_1 x_1 + \beta_2 x_2 + \cdots + \beta_p x_p +``` + +The inverse link function maps from the linear predictor back to the mean of the response: $\mu = g^{-1}(\eta)$. Each link function corresponds to a distribution family: + +| Link Function | $g(\mu)$ | $g^{-1}(\eta)$ | Distribution Family | +|---|---|---|---| +| Identity | $\mu$ | $\eta$ | Normal | +| Log | $\log(\mu)$ | $\exp(\eta)$ | Poisson | +| Logit | $\log\!\left(\frac{\mu}{1-\mu}\right)$ | $\frac{1}{1+\exp(-\eta)}$ | Binomial | +| Probit | $\Phi^{-1}(\mu)$ | $\Phi(\eta)$ | Binomial | +| CLogLog | $\log(-\log(1-\mu))$ | $1-\exp(-\exp(\eta))$ | Binomial | + +**Parameter estimation.** Model parameters $\beta$ are estimated by maximum likelihood estimation (MLE), which maximizes the total log-likelihood over all $n$ observations: + +```math +\hat{\beta} = \arg\max_{\beta} \sum_{i=1}^{n} \ell(\mu_i, y_i) +``` + +where the individual log-likelihood contribution $\ell(\mu_i, y_i)$ depends on the distribution family: + +- **Normal family** (Identity link): + +```math +\ell(\mu_i, y_i) = -\tfrac{1}{2}(y_i - \mu_i)^2 +``` + +- **Poisson family** (Log link): + +```math +\ell(\mu_i, y_i) = y_i \log(\mu_i) - \mu_i +``` + +- **Binomial family** (Logit, Probit, or CLogLog link): + +```math +\ell(\mu_i, y_i) = y_i \log(\mu_i) + (1 - y_i)\log(1 - \mu_i) +``` + +**Model selection.** The Akaike Information Criterion (AIC) balances goodness of fit against model complexity: + +```math +\text{AIC} = 2k - 2\ln(\hat{L}) +``` + +where $k$ is the number of estimated parameters and $\hat{L}$ is the maximized likelihood. The corrected AIC (AICc) adjusts for small sample sizes, and the Bayesian Information Criterion (BIC) applies a stronger penalty for additional parameters. + +```cs +using Numerics.MachineLearning; +using Numerics.Mathematics.LinearAlgebra; +using Numerics.Mathematics.Optimization; +using Numerics.Functions; + +// Training data (no intercept column needed — GLM adds it automatically when hasIntercept = true) +double[,] X = { + { 2.5, 1.2 }, // Observation 1: [feature1, feature2] + { 3.1, 1.5 }, + { 2.8, 1.1 }, + { 3.5, 1.8 }, + { 2.2, 0.9 } +}; + +double[] y = { 45.2, 52.3, 47.8, 58.1, 42.5 }; // Response variable + +// Create GLM +var glm = new GeneralizedLinearModel( + x: new Matrix(X), + y: new Vector(y), + hasIntercept: true, // Adds intercept column automatically + linkType: LinkFunctionType.Identity // Link function type +); + +// Set optimizer (optional) +glm.SetOptimizer(LocalMethod.NelderMead); + +// Train model +glm.Train(); + +Console.WriteLine("GLM Results:"); +Console.WriteLine($"Parameters: [{string.Join(", ", glm.Parameters.Select(p => p.ToString("F4")))}]"); +Console.WriteLine($"Standard Errors: [{string.Join(", ", glm.ParameterStandardErrors.Select(se => se.ToString("F4")))}]"); +Console.WriteLine($"p-values: [{string.Join(", ", glm.ParameterPValues.Select(p => p.ToString("F4")))}]"); + +// Model selection criteria +Console.WriteLine($"\nModel Selection:"); +Console.WriteLine($" AIC: {glm.AIC:F2}"); +Console.WriteLine($" BIC: {glm.BIC:F2}"); +Console.WriteLine($" Standard Error: {glm.StandardError:F4}"); + +// Make predictions +double[,] XNew = { + { 3.0, 1.4 }, + { 2.6, 1.0 } +}; + +double[] predictions = glm.Predict(new Matrix(XNew)); + +Console.WriteLine($"\nPredictions:"); +for (int i = 0; i < predictions.Length; i++) +{ + Console.WriteLine($" X_new[{i}] → {predictions[i]:F2}"); +} + +// Prediction intervals (alpha = 0.1 for 90% interval) +double[,] intervals = glm.Predict(new Matrix(XNew), alpha: 0.1); + +Console.WriteLine($"\n90% Prediction Intervals:"); +for (int i = 0; i < XNew.GetLength(0); i++) +{ + Console.WriteLine($" X_new[{i}]: [{intervals[i, 0]:F2}, {intervals[i, 2]:F2}]"); +} +``` + +**Link Function Types** (`LinkFunctionType` in `Numerics.Functions`): +- `LinkFunctionType.Identity` - g(μ) = μ (Normal/Gaussian family) +- `LinkFunctionType.Log` - g(μ) = log(μ) (Poisson family) +- `LinkFunctionType.Logit` - g(μ) = log(μ/(1-μ)) (Binomial family) +- `LinkFunctionType.Probit` - g(μ) = Φ⁻¹(μ) (Binomial family, alternative) +- `LinkFunctionType.ComplementaryLogLog` - g(μ) = log(-log(1-μ)) (Asymmetric binary response) + +### Decision Trees + +Classification and regression trees [[2]](#2) recursively partition the feature space using binary splits of the form $x_j \leq t$ (left child) and $x_j > t$ (right child), where $x_j$ is a feature and $t$ is a threshold. At each internal node, the algorithm selects the feature and threshold that maximize a splitting criterion. + +**Classification trees** use information gain based on Shannon entropy. The entropy of a node $S$ with $C$ classes is: + +```math +H(S) = -\sum_{c=1}^{C} p_c \log(p_c) +``` + +where $p_c$ is the proportion of observations belonging to class $c$. The information gain from splitting node $S$ into children $S_L$ and $S_R$ is: + +```math +\text{IG}(S, S_L, S_R) = H(S) - \frac{n_L}{n_S} H(S_L) - \frac{n_R}{n_S} H(S_R) +``` + +where $n_L$, $n_R$, and $n_S$ are the number of observations in the left child, right child, and parent node, respectively. + +**Regression trees** use variance reduction. The population variance of a node $S$ is: + +```math +\text{Var}(S) = \frac{1}{n_S} \sum_{i \in S} (y_i - \bar{y}_S)^2 +``` + +The variance reduction from a split is: + +```math +\text{VR}(S, S_L, S_R) = \text{Var}(S) - \frac{n_L}{n_S} \text{Var}(S_L) - \frac{n_R}{n_S} \text{Var}(S_R) +``` + +**Leaf node predictions.** For regression, the prediction is the mean of the training observations at the leaf: $\hat{y} = \bar{y}_{\text{leaf}}$. For classification, the prediction is the most frequent class among the leaf observations. + +**Stopping criteria.** Tree growth stops when any of the following conditions is met: the maximum depth (`MaxDepth`, default 100) is reached, the number of observations at a node falls below `MinimumSplitSize` (default 2), or all observations at a node belong to the same class. + +```cs +using Numerics.MachineLearning; + +// Classification example (Iris-like data, requires at least 10 training samples) +double[,] X = { + { 5.1, 3.5, 1.4, 0.2 }, // Iris features: sepal length, sepal width, petal length, petal width + { 4.9, 3.0, 1.4, 0.2 }, + { 4.7, 3.2, 1.3, 0.2 }, + { 5.0, 3.6, 1.4, 0.2 }, + { 7.0, 3.2, 4.7, 1.4 }, + { 6.4, 3.2, 4.5, 1.5 }, + { 6.9, 3.1, 4.9, 1.5 }, + { 6.3, 3.3, 6.0, 2.5 }, + { 5.8, 2.7, 5.1, 1.9 }, + { 7.1, 3.0, 5.9, 2.1 }, + { 6.5, 3.0, 5.8, 2.2 } +}; + +double[] y = { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2 }; // Classes: Setosa(0), Versicolor(1), Virginica(2) + +// Create and train decision tree +var tree = new DecisionTree(X, y); +tree.MaxDepth = 5; // Optional: limit tree depth (default: 100) +tree.Train(); + +Console.WriteLine($"Decision Tree Trained: {tree.IsTrained}"); + +// Predict single sample (pass as 2D array with 1 row for multi-feature input) +double[,] testSample = { { 5.0, 3.0, 1.6, 0.2 } }; +double[] prediction = tree.Predict(testSample); +Console.WriteLine($"Prediction for test sample: Class {prediction[0]}"); + +// Predict multiple samples +double[,] testSamples = { + { 5.0, 3.0, 1.6, 0.2 }, + { 6.0, 3.0, 4.5, 1.5 }, + { 6.5, 3.0, 5.5, 2.0 } +}; + +double[] predictions = tree.Predict(testSamples); + +Console.WriteLine("\nBatch predictions:"); +for (int i = 0; i < predictions.Length; i++) +{ + Console.WriteLine($" Sample {i}: Class {predictions[i]}"); +} +``` + +### Random Forests + +Ensemble of decision trees for improved accuracy [[3]](#3) [[9]](#9). Random forests use bootstrap aggregation (bagging) to reduce variance and improve generalization. Given a training set of $n$ observations, the algorithm builds $B$ independent decision trees (default $B = 1000$), each trained on a bootstrap sample drawn with replacement from the original data. + +**Bootstrap sampling.** For each tree $b = 1, \ldots, B$, draw $n$ samples with replacement from the original $n$ observations. On average, each bootstrap sample contains approximately $1 - 1/e \approx 63.2\%$ of the unique training observations, leaving the remainder as out-of-bag (OOB) observations that can be used for validation. + +**Feature subsampling.** At each split within a tree, a random subset of features is considered as split candidates. The `Features` property controls the number of candidate features (default $d - 1$, where $d$ is the total number of features). This decorrelates the trees and further reduces variance. + +**Aggregation.** The ensemble prediction for a new input $\mathbf{x}$ aggregates the predictions of all $B$ trees: + +```math +\hat{f}_{\text{bag}}(\mathbf{x}) = \frac{1}{B} \sum_{b=1}^{B} \hat{f}_b(\mathbf{x}) +``` + +For regression, predictions are the direct percentiles across all tree outputs. For classification, predictions are the floor of the percentiles. The `Predict` method returns lower, median, upper, and mean values across the ensemble, providing built-in prediction uncertainty quantification. + +```cs +using Numerics.MachineLearning; + +double[,] X = { + // Same Iris-like data as above (requires at least 10 training samples) + { 5.1, 3.5, 1.4, 0.2 }, + { 4.9, 3.0, 1.4, 0.2 }, + { 4.7, 3.2, 1.3, 0.2 }, + { 5.0, 3.6, 1.4, 0.2 }, + { 7.0, 3.2, 4.7, 1.4 }, + { 6.4, 3.2, 4.5, 1.5 }, + { 6.9, 3.1, 4.9, 1.5 }, + { 6.3, 3.3, 6.0, 2.5 }, + { 5.8, 2.7, 5.1, 1.9 }, + { 7.1, 3.0, 5.9, 2.1 }, + { 6.5, 3.0, 5.8, 2.2 } +}; + +double[] y = { 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2 }; + +// Create and train random forest +var forest = new RandomForest(X, y, seed: 12345); +forest.NumberOfTrees = 100; // Default: 1000 +forest.MaxDepth = 5; // Default: 100 +forest.Train(); + +Console.WriteLine($"Random Forest Trained: {forest.IsTrained}"); +Console.WriteLine($"Number of trees: {forest.NumberOfTrees}"); + +// Predict with confidence intervals (pass as 2D array for multi-feature input) +// Predict returns double[,] with columns: lower(0), median(1), upper(2), mean(3) +double[,] testSample = { { 5.0, 3.0, 1.6, 0.2 } }; +double[,] result = forest.Predict(testSample, alpha: 0.1); // 90% CI + +Console.WriteLine($"\nPrediction:"); +Console.WriteLine($" Predicted class (median): {result[0, 1]:F0}"); +Console.WriteLine($" Mean: {result[0, 3]:F2}"); +Console.WriteLine($" 90% CI: [{result[0, 0]:F2}, {result[0, 2]:F2}]"); + +// Batch prediction +double[,] testSamples = { + { 5.0, 3.0, 1.6, 0.2 }, + { 6.0, 3.0, 4.5, 1.5 } +}; + +double[,] results = forest.Predict(testSamples, alpha: 0.1); + +Console.WriteLine($"\nBatch predictions:"); +for (int i = 0; i < testSamples.GetLength(0); i++) +{ + Console.WriteLine($" Sample {i}: Class {results[i, 1]:F0}, " + + $"CI [{results[i, 0]:F2}, {results[i, 2]:F2}]"); +} +``` + +**Advantages of Random Forests:** +- Reduces overfitting compared to single tree +- Provides prediction uncertainty +- Handles missing values well +- Works with mixed feature types + +### k-Nearest Neighbors (KNN) + +Non-parametric classification and regression [[4]](#4). KNN is a lazy learning algorithm that stores the training data and defers computation until prediction time. Given a new input $\mathbf{x}$, the algorithm finds the $k$ nearest training observations using the Euclidean distance: + +```math +d(\mathbf{x}, \mathbf{x}_i) = \sqrt{\sum_{j=1}^{p} (x_j - x_{ij})^2} +``` + +where $p$ is the number of features. + +**Regression prediction.** For regression, the prediction is an inverse distance weighted average of the $k$ nearest neighbors: + +```math +\hat{y} = \frac{\sum_{i=1}^{k} w_i \cdot y_i}{\sum_{i=1}^{k} w_i}, \quad w_i = \frac{1}{d(\mathbf{x}, \mathbf{x}_i)^2} +``` + +If the distance to a neighbor is exactly zero (i.e., the query point coincides with a training point), that neighbor receives a weight of $w_i = 1$ and all other weights are set accordingly. + +**Classification prediction.** For classification, the prediction is determined by majority vote among the $k$ nearest neighbors: the class that appears most frequently is assigned to the new observation. + +**Choosing $k$.** The choice of $k$ controls the bias-variance tradeoff. A small $k$ produces a flexible model sensitive to noise, while a large $k$ produces smoother decision boundaries at the cost of increased bias. A common rule of thumb is $k = \sqrt{n}$, where $n$ is the number of training observations, though cross-validation is preferred for rigorous selection. + +**Curse of dimensionality.** As the number of features $p$ grows, Euclidean distances become increasingly uniform, degrading the discriminative power of nearest neighbor methods. Feature scaling (normalization) is critical because KNN is sensitive to the relative magnitudes of features. + +```cs +using Numerics.MachineLearning; + +double[,] X = { + { 1.0, 2.0 }, + { 1.5, 1.8 }, + { 1.0, 0.6 }, + { 2.0, 1.5 }, + { 1.2, 1.0 }, + { 5.0, 8.0 }, + { 8.0, 8.0 }, + { 9.0, 11.0 }, + { 7.0, 9.0 }, + { 6.5, 7.5 } +}; + +double[] y = { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 }; // Binary classification + +// Create KNN classifier (no explicit training needed — lazy learner) +var knn = new KNearestNeighbors(X, y, k: 3); + +// Predict single sample +double[,] testPoint = { { 2.0, 3.0 } }; +double[] prediction = knn.Predict(testPoint); +Console.WriteLine($"KNN Prediction for [{testPoint[0, 0]}, {testPoint[0, 1]}]: Class {prediction[0]}"); + +// Predict multiple samples +double[,] testPoints = { { 2.0, 3.0 }, { 7.0, 8.0 } }; +double[] predictions = knn.Predict(testPoints); + +Console.WriteLine($"Batch predictions:"); +for (int i = 0; i < predictions.Length; i++) +{ + Console.WriteLine($" Point {i}: Class {predictions[i]}"); +} +``` + +**Distance Metrics:** +- Euclidean (default) +- Manhattan +- Minkowski + +**Choosing k:** +- Small k: More sensitive to noise +- Large k: Smoother boundaries +- Rule of thumb: k = √n or use cross-validation + +### Naive Bayes + +Probabilistic classifier based on Bayes' theorem [[5]](#5). The Naive Bayes classifier applies Bayes' theorem with the assumption of conditional independence among features. Given a feature vector $\mathbf{x} = (x_1, x_2, \ldots, x_p)$, the posterior probability of class $c$ is: + +```math +P(c \mid \mathbf{x}) \propto P(c) \cdot P(\mathbf{x} \mid c) +``` + +The conditional independence assumption simplifies the class-conditional likelihood to a product over individual features: + +```math +P(\mathbf{x} \mid c) = \prod_{j=1}^{p} P(x_j \mid c) +``` + +This implementation uses Gaussian Naive Bayes, where each feature conditional on the class follows a normal distribution: + +```math +P(x_j \mid c) = \mathcal{N}(x_j \mid \mu_{c,j},\, \sigma_{c,j}) = \frac{1}{\sqrt{2\pi}\,\sigma_{c,j}} \exp\!\left(-\frac{(x_j - \mu_{c,j})^2}{2\sigma_{c,j}^2}\right) +``` + +where $\mu_{c,j}$ and $\sigma_{c,j}$ are the mean and standard deviation of feature $j$ within class $c$, estimated from the training data. + +**Prior probabilities.** The class prior $P(c)$ is estimated as the relative frequency of class $c$ in the training data: + +```math +P(c) = \frac{n_c}{n} +``` + +where $n_c$ is the number of training observations in class $c$ and $n$ is the total number of observations. + +**Classification rule.** The predicted class is determined by Maximum A Posteriori (MAP) estimation. Working in log-space for numerical stability: + +```math +\hat{c} = \arg\max_{c} \left[ \log P(c) + \sum_{j=1}^{p} \log \mathcal{N}(x_j \mid \mu_{c,j},\, \sigma_{c,j}) \right] +``` + +```cs +using Numerics.MachineLearning; + +// Text classification example (word counts, requires at least 10 training samples) +double[,] X = { + { 2, 1, 0, 1 }, // Document 1: word counts + { 1, 1, 1, 0 }, + { 3, 2, 0, 1 }, + { 2, 0, 1, 0 }, + { 1, 2, 0, 2 }, + { 0, 3, 2, 1 }, + { 1, 0, 1, 2 }, + { 0, 1, 3, 2 }, + { 1, 0, 2, 3 }, + { 0, 2, 2, 1 } +}; + +double[] y = { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 }; // Classes: spam(1), ham(0) + +// Create and train Naive Bayes +var nb = new NaiveBayes(X, y); +nb.Train(); + +Console.WriteLine($"Naive Bayes trained: {nb.IsTrained}"); + +// Predict single sample +double[,] testDoc = { { 1, 2, 0, 1 } }; +double[] prediction = nb.Predict(testDoc); +Console.WriteLine($"Prediction: Class {prediction[0]}"); + +// Predict multiple samples +double[,] testDocs = { { 1, 2, 0, 1 }, { 0, 3, 1, 0 } }; +double[] predictions = nb.Predict(testDocs); +for (int i = 0; i < predictions.Length; i++) + Console.WriteLine($" Doc {i}: Class {predictions[i]}"); +``` + +**Assumptions:** +- Features are conditionally independent given class +- Works well despite violation of independence +- Fast training and prediction +- Good for text classification + +--- + +## Unsupervised Learning + +### k-Means Clustering + +Partition data into $k$ clusters [[6]](#6). The $k$-means algorithm seeks to minimize the within-cluster sum of squares (WCSS) objective function: + +```math +J = \sum_{k=1}^{K} \sum_{\mathbf{x} \in C_k} \|\mathbf{x} - \boldsymbol{\mu}_k\|^2 +``` + +where $C_k$ is the set of observations assigned to cluster $k$ and $\boldsymbol{\mu}_k$ is the centroid (mean) of cluster $k$. + +**Lloyd's algorithm.** The algorithm alternates between two steps until convergence: + +1. **E-step (assignment):** Assign each observation $\mathbf{x}_i$ to the nearest centroid using Euclidean distance: + +```math +c_i = \arg\min_{k} \|\mathbf{x}_i - \boldsymbol{\mu}_k\|^2 +``` + +2. **M-step (update):** Recompute each centroid as the mean of all observations assigned to its cluster: + +```math +\boldsymbol{\mu}_k = \frac{1}{|C_k|} \sum_{\mathbf{x} \in C_k} \mathbf{x} +``` + +The algorithm converges when the cluster labels do not change between iterations, or when `MaxIterations` (default 1000) is reached. + +**$k$-Means++ initialization** [[11]](#11). The default initialization method selects initial centroids that are well-separated. The first centroid is chosen uniformly at random. Each subsequent centroid is selected with probability proportional to $D(\mathbf{x})^2$, where $D(\mathbf{x})$ is the Euclidean distance from $\mathbf{x}$ to the nearest already-chosen centroid. This initialization significantly reduces the chance of poor convergence compared to random initialization. + +```cs +using Numerics.MachineLearning; + +// 2D data points +double[,] X = { + { 1.0, 2.0 }, + { 1.5, 1.8 }, + { 5.0, 8.0 }, + { 8.0, 8.0 }, + { 1.0, 0.6 }, + { 9.0, 11.0 }, + { 8.0, 2.0 }, + { 10.0, 2.0 }, + { 9.0, 3.0 } +}; + +// Create k-means with 3 clusters +var kmeans = new KMeans(X, k: 3); +kmeans.MaxIterations = 100; + +// Train (use seed for reproducibility, k-means++ initialization by default) +kmeans.Train(seed: 12345); + +Console.WriteLine($"k-Means Clustering (k={kmeans.K}):"); +Console.WriteLine($"Iterations: {kmeans.Iterations}"); + +// Cluster centers +Console.WriteLine($"\nCluster Centers:"); +for (int i = 0; i < kmeans.K; i++) +{ + Console.WriteLine($" Cluster {i}: [{kmeans.Means[i, 0]:F2}, {kmeans.Means[i, 1]:F2}]"); +} + +// Cluster labels +Console.WriteLine($"\nCluster Assignments:"); +for (int i = 0; i < X.GetLength(0); i++) +{ + Console.WriteLine($" Point [{X[i, 0]:F1}, {X[i, 1]:F1}] → Cluster {kmeans.Labels[i]}"); +} + +// Cluster sizes +var clusterSizes = kmeans.Labels.GroupBy(l => l).Select(g => g.Count()).ToArray(); +Console.WriteLine($"\nCluster sizes: [{string.Join(", ", clusterSizes)}]"); +``` + +**Choosing k:** +- Elbow method (plot inertia vs. k) +- Silhouette analysis +- Domain knowledge + +**Initialization Methods:** +- Random selection +- k-means++ (default, better initialization) + +### Gaussian Mixture Models (GMM) + +Probabilistic clustering with soft assignments [[7]](#7) [[10]](#10). A Gaussian Mixture Model represents the data as a mixture of $K$ multivariate Gaussian components. The probability density function of the mixture is: + +```math +p(\mathbf{x}) = \sum_{k=1}^{K} \pi_k \cdot \mathcal{N}(\mathbf{x} \mid \boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k) +``` + +where $\pi_k$ are the mixing weights (with $\sum_k \pi_k = 1$ and $\pi_k \geq 0$), $\boldsymbol{\mu}_k$ is the mean vector, and $\boldsymbol{\Sigma}_k$ is the covariance matrix of component $k$. + +**Expectation-Maximization (EM) algorithm.** The parameters are estimated by maximizing the log-likelihood using the EM algorithm, initialized from a $k$-means solution with equal weights and near-identity covariance matrices. + +The **E-step** computes the responsibility of each component $k$ for each observation $\mathbf{x}_i$: + +```math +r_{ik} = \frac{\pi_k \cdot \mathcal{N}(\mathbf{x}_i \mid \boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k)}{\sum_{j=1}^{K} \pi_j \cdot \mathcal{N}(\mathbf{x}_i \mid \boldsymbol{\mu}_j, \boldsymbol{\Sigma}_j)} +``` + +The implementation uses the log-sum-exp trick with Cholesky decomposition for numerical stability. + +The **M-step** updates the parameters using the responsibilities: + +```math +\pi_k = \frac{1}{n}\sum_{i=1}^{n} r_{ik}, \qquad \boldsymbol{\mu}_k = \frac{\sum_{i=1}^{n} r_{ik}\,\mathbf{x}_i}{\sum_{i=1}^{n} r_{ik}}, \qquad \boldsymbol{\Sigma}_k = \frac{\sum_{i=1}^{n} r_{ik}\,(\mathbf{x}_i - \boldsymbol{\mu}_k)(\mathbf{x}_i - \boldsymbol{\mu}_k)^\top}{\sum_{i=1}^{n} r_{ik}} +``` + +**Convergence.** The algorithm iterates until the relative change in log-likelihood falls below `Tolerance` (default $10^{-8}$) or `MaxIterations` (default 1000) is reached. The total log-likelihood is: + +```math +\mathcal{L} = \sum_{i=1}^{n} \log\!\left(\sum_{k=1}^{K} \pi_k \cdot \mathcal{N}(\mathbf{x}_i \mid \boldsymbol{\mu}_k, \boldsymbol{\Sigma}_k)\right) +``` + +**Connection to $k$-means.** GMM generalizes $k$-means: when all covariance matrices are constrained to $\sigma^2 \mathbf{I}$ and $\sigma^2 \to 0$, the soft responsibilities $r_{ik}$ converge to hard 0/1 assignments, recovering the $k$-means solution. + +```cs +using Numerics.MachineLearning; + +double[,] X = { + // Same data as k-means example + { 1.0, 2.0 }, { 1.5, 1.8 }, { 5.0, 8.0 }, + { 8.0, 8.0 }, { 1.0, 0.6 }, { 9.0, 11.0 } +}; + +// Create GMM with 2 components +var gmm = new GaussianMixtureModel(X, k: 2); +gmm.MaxIterations = 100; +gmm.Tolerance = 1e-3; + +// Train using EM algorithm +gmm.Train(seed: 12345); + +Console.WriteLine($"GMM Clustering ({gmm.K} components):"); +Console.WriteLine($"Iterations: {gmm.Iterations}"); +Console.WriteLine($"Log-likelihood: {gmm.LogLikelihood:F2}"); + +// Component parameters +Console.WriteLine($"\nComponent Parameters:"); +for (int i = 0; i < gmm.K; i++) +{ + Console.WriteLine($" Component {i}:"); + Console.WriteLine($" Weight: {gmm.Weights[i]:F3}"); + int dims = X.GetLength(1); + Console.Write($" Mean: ["); + for (int d = 0; d < dims; d++) + Console.Write($"{gmm.Means[i, d]:F2}{(d < dims - 1 ? ", " : "")}"); + Console.WriteLine("]"); +} + +// Cluster labels +Console.WriteLine($"\nCluster Assignments:"); +for (int i = 0; i < X.GetLength(0); i++) +{ + Console.WriteLine($" Point [{X[i, 0]:F1}, {X[i, 1]:F1}] → Component {gmm.Labels[i]}"); +} +``` + +**Advantages over k-Means:** +- Soft clustering (probabilistic assignments) +- Flexible cluster shapes (elliptical vs. spherical) +- Provides uncertainty quantification +- Can model overlapping clusters + +### Jenks Natural Breaks + +Optimal classification for univariate data [[8]](#8). The Jenks Natural Breaks algorithm (also known as the Fisher-Jenks algorithm) partitions a sorted univariate dataset into $k$ classes by minimizing the within-class sum of squared deviations while maximizing the between-class separation. The quality of a classification is measured by the Goodness of Variance Fit (GVF): + +```math +\text{GVF} = \frac{\text{SDAM} - \text{SDCM}}{\text{SDAM}} +``` + +where SDAM is the sum of squared deviations from the array (overall) mean: + +```math +\text{SDAM} = \sum_{i=1}^{n} (x_i - \bar{x})^2 +``` + +and SDCM is the sum of squared deviations from the class means: + +```math +\text{SDCM} = \sum_{c=1}^{k} \sum_{x \in C_c} (x - \bar{x}_c)^2 +``` + +A GVF of 1.0 indicates a perfect fit (zero within-class variance), while values closer to 0 indicate poor classification. The algorithm uses dynamic programming to efficiently find the optimal break points that maximize the GVF, avoiding the exponential cost of exhaustive search over all possible partitions. + +```cs +using Numerics.MachineLearning; + +// Data values (e.g., elevation, rainfall, etc.) +double[] data = { 10, 12, 15, 18, 22, 25, 28, 35, 40, 45, 50, 55, 60, 70, 80 }; + +// Find natural breaks with 4 classes (computation happens in constructor) +int nClasses = 4; +var jenks = new JenksNaturalBreaks(data, nClasses); + +Console.WriteLine($"Jenks Natural Breaks ({nClasses} classes):"); +Console.WriteLine($"Break points: [{string.Join(", ", jenks.Breaks.Select(b => b.ToString("F1")))}]"); +Console.WriteLine($"Goodness of variance fit: {jenks.GoodnessOfVarianceFit:F4}"); + +// Access cluster details +Console.WriteLine($"\nCluster details:"); +for (int c = 0; c < jenks.Clusters.Length; c++) +{ + var cluster = jenks.Clusters[c]; + Console.WriteLine($" Cluster {c}: {cluster.Count} values"); +} +``` + +**Applications:** +- Choropleth map classification +- Data binning for visualization +- Natural grouping identification +- Minimizes within-class variance + +--- + +## Practical Examples + +### Example 1: Regression with GLM + +```cs +using Numerics.MachineLearning; +using Numerics.Mathematics.LinearAlgebra; +using Numerics.Functions; + +// Predict home prices (no intercept column — GLM adds it automatically) +double[,] features = { + { 1500, 3, 20 }, // [sqft, bedrooms, age] + { 1800, 4, 15 }, + { 1200, 2, 30 }, + { 2000, 4, 10 }, + { 1600, 3, 25 } +}; + +double[] prices = { 250000, 320000, 190000, 380000, 270000 }; // $ + +var glm = new GeneralizedLinearModel( + new Matrix(features), + new Vector(prices), + hasIntercept: true, + linkType: LinkFunctionType.Identity +); + +glm.Train(); + +Console.WriteLine("Home Price Prediction Model:"); +Console.WriteLine($"Coefficients:"); +Console.WriteLine($" Intercept: ${glm.Parameters[0]:F0}"); +Console.WriteLine($" Per sqft: ${glm.Parameters[1]:F2}"); +Console.WriteLine($" Per bedroom: ${glm.Parameters[2]:F0}"); +Console.WriteLine($" Per year age: ${glm.Parameters[3]:F0}"); + +// Predict new home +double[,] newHome = { { 1700, 3, 12 } }; +double predicted = glm.Predict(new Matrix(newHome))[0]; +// Predict returns columns: lower(0), mean(1), upper(2) +double[,] interval = glm.Predict(new Matrix(newHome), alpha: 0.1); + +Console.WriteLine($"\nPrediction for 1700 sqft, 3BR, 12 years:"); +Console.WriteLine($" Predicted price: ${predicted:F0}"); +Console.WriteLine($" 90% Interval: [${interval[0, 0]:F0}, ${interval[0, 2]:F0}]"); +``` + +### Example 2: Classification Pipeline + +```cs +// Sample binary classification data (requires at least 10 training samples) +double[,] X_train = { + { 2.5, 3.2 }, { 3.1, 2.8 }, { 2.8, 3.5 }, { 3.3, 2.9 }, { 2.6, 3.1 }, // Class 0 + { 6.2, 5.8 }, { 5.9, 6.1 }, { 6.5, 5.5 }, { 5.8, 6.3 }, { 6.1, 5.7 } // Class 1 +}; +double[] y_train = { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1 }; + +double[,] X_test = { + { 2.9, 3.0 }, { 6.0, 5.9 } // One from each class +}; +double[] y_test = { 0, 1 }; + +// Train random forest (no nTrees constructor parameter — set via property) +var rf = new RandomForest(X_train, y_train, seed: 42); +rf.NumberOfTrees = 100; +rf.Train(); + +// Evaluate — Predict returns double[,] with columns: lower(0), median(1), upper(2), mean(3) +double[,] predictions = rf.Predict(X_test); +int correct = 0; +for (int i = 0; i < y_test.Length; i++) +{ + if (predictions[i, 1] == y_test[i]) // Use median (column 1) as predicted class + correct++; +} + +double accuracy = (double)correct / y_test.Length; + +Console.WriteLine($"Random Forest Classification:"); +Console.WriteLine($" Accuracy: {accuracy:P1}"); +Console.WriteLine($" Correct: {correct}/{y_test.Length}"); +``` + +### Example 3: Customer Segmentation + +```cs +// Customer data: [annual_spending, visit_frequency, avg_basket_size] +double[,] customers = { + { 1200, 24, 50 }, // Regular customer + { 5000, 52, 95 }, // High-value customer + { 300, 6, 45 }, // Occasional customer + { 4800, 48, 100 }, // High-value customer + { 800, 12, 65 }, // Regular customer + { 250, 4, 55 }, // Occasional customer + { 6000, 60, 105 } // VIP customer +}; + +// Cluster into 3 segments +var kmeans = new KMeans(customers, k: 3); +kmeans.Train(seed: 42); + +Console.WriteLine("Customer Segmentation:"); +for (int i = 0; i < 3; i++) +{ + var segment = Enumerable.Range(0, customers.GetLength(0)) + .Where(j => kmeans.Labels[j] == i) + .ToArray(); + + Console.WriteLine($"\nSegment {i} ({segment.Length} customers):"); + Console.WriteLine($" Avg spending: ${segment.Average(j => customers[j, 0]):F0}"); + Console.WriteLine($" Avg visits: {segment.Average(j => customers[j, 1]):F0}/year"); + Console.WriteLine($" Avg basket: ${segment.Average(j => customers[j, 2]):F0}"); +} +``` + +### Example 4: Flood Damage Prediction with Logistic GLM + +Predicting whether flood damage occurs based on hydraulic variables using data from [`example-data/flood-damage-glm.csv`](example-data/flood-damage-glm.csv): + +```cs +using System.IO; +using System.Linq; +using Numerics.MachineLearning; +using Numerics.Functions; +using Numerics.Mathematics.LinearAlgebra; + +// Load CSV data (skip comment lines starting with #) +string[] lines = File.ReadAllLines("example-data/flood-damage-glm.csv"); +var dataLines = lines + .Where(line => !line.StartsWith("#") && !string.IsNullOrWhiteSpace(line)) + .Skip(1) // Skip header + .ToArray(); + +int n = dataLines.Length; +double[,] features = new double[n, 3]; +double[] damage = new double[n]; + +for (int i = 0; i < n; i++) +{ + var parts = dataLines[i].Split(','); + features[i, 0] = double.Parse(parts[0]); // FloodStage_ft + features[i, 1] = double.Parse(parts[1]); // Duration_hr + features[i, 2] = double.Parse(parts[2]); // Velocity_fps + damage[i] = double.Parse(parts[3]); // DamageOccurred (0/1) +} + +// Fit logistic regression (GLM with Logit link) +var glm = new GeneralizedLinearModel( + new Matrix(features), + new Vector(damage), + hasIntercept: true, + linkType: LinkFunctionType.Logit +); + +glm.Train(); + +// Print R-style summary +Console.WriteLine("Flood Damage Logistic Regression"); +Console.WriteLine("=" + new string('=', 50)); +foreach (var line in glm.Summary()) +{ + Console.WriteLine(line); +} + +// Model fit statistics +Console.WriteLine($"\nAIC: {glm.AIC:F2}"); +Console.WriteLine($"AICc: {glm.AICc:F2}"); +Console.WriteLine($"BIC: {glm.BIC:F2}"); + +// Predict damage probability for a new flood event +double[,] newEvent = { { 14.0, 10, 3.5 } }; // Stage=14ft, Duration=10hr, Velocity=3.5fps +double[] prob = glm.Predict(new Matrix(newEvent)); +Console.WriteLine($"\nPredicted damage probability: {prob[0]:P1}"); + +// Prediction with 90% confidence interval +double[,] interval = glm.Predict(new Matrix(newEvent), alpha: 0.1); +Console.WriteLine($"90% Prediction interval: [{interval[0, 0]:P1}, {interval[0, 2]:P1}]"); +Console.WriteLine($" Mean prediction: {interval[0, 1]:P1}"); + +// Compare link functions using AIC +var linkTypes = new[] { + LinkFunctionType.Logit, + LinkFunctionType.Probit, + LinkFunctionType.ComplementaryLogLog +}; + +Console.WriteLine("\nLink Function Comparison:"); +Console.WriteLine(" Link | AIC | BIC"); +Console.WriteLine(" ------------------|---------|--------"); + +foreach (var link in linkTypes) +{ + var model = new GeneralizedLinearModel( + new Matrix(features), + new Vector(damage), + hasIntercept: true, + linkType: link + ); + model.Train(); + Console.WriteLine($" {link,-18} | {model.AIC,7:F2} | {model.BIC,7:F2}"); +} +``` + +## Model Selection and Evaluation + +### Cross-Validation + +```cs +using Numerics.Data.Statistics; + +// Simple k-fold cross-validation +int k = 5; +int n = X.GetLength(0); +int dims = X.GetLength(1); +int foldSize = n / k; + +double[] accuracies = new double[k]; + +for (int fold = 0; fold < k; fold++) +{ + // Split data into train/test indices + var testIndices = Enumerable.Range(fold * foldSize, foldSize).ToArray(); + var trainIndices = Enumerable.Range(0, n) + .Where(i => !testIndices.Contains(i)) + .ToArray(); + + // Extract train/test data by copying rows + double[,] X_trainFold = new double[trainIndices.Length, dims]; + double[] y_trainFold = new double[trainIndices.Length]; + for (int i = 0; i < trainIndices.Length; i++) + { + for (int d = 0; d < dims; d++) + X_trainFold[i, d] = X[trainIndices[i], d]; + y_trainFold[i] = y[trainIndices[i]]; + } + + double[,] X_testFold = new double[testIndices.Length, dims]; + double[] y_testFold = new double[testIndices.Length]; + for (int i = 0; i < testIndices.Length; i++) + { + for (int d = 0; d < dims; d++) + X_testFold[i, d] = X[testIndices[i], d]; + y_testFold[i] = y[testIndices[i]]; + } + + // Train and evaluate + var model = new DecisionTree(X_trainFold, y_trainFold); + model.Train(); + var predictions = model.Predict(X_testFold); + + int correct = 0; + for (int i = 0; i < testIndices.Length; i++) + if (predictions[i] == y_testFold[i]) correct++; + + accuracies[fold] = (double)correct / testIndices.Length; +} + +Console.WriteLine($"Cross-Validation Results:"); +Console.WriteLine($" Mean accuracy: {accuracies.Average():P1}"); +Console.WriteLine($" Std dev: {Statistics.StandardDeviation(accuracies):F4}"); +``` + +### Model Comparison + +```cs +// Compare different classifiers on the same dataset +double[,] X = { /* training features (at least 10 rows) */ }; +double[] y = { /* training labels */ }; +double[,] X_test = { /* test features */ }; +double[] y_test = { /* test labels */ }; + +// Train models (KNN is lazy — no Train() call needed) +var decisionTree = new DecisionTree(X, y); +decisionTree.Train(); + +var randomForest = new RandomForest(X, y, seed: 42); +randomForest.NumberOfTrees = 50; +randomForest.Train(); + +var knn = new KNearestNeighbors(X, y, k: 3); + +// Evaluate each model +// Note: DecisionTree and KNN return double[], RandomForest returns double[,] +Console.WriteLine("Model Comparison:"); + +// DecisionTree returns double[] +double[] dtPredictions = decisionTree.Predict(X_test); +int dtCorrect = 0; +for (int i = 0; i < y_test.Length; i++) + if (dtPredictions[i] == y_test[i]) dtCorrect++; +Console.WriteLine($" Decision Tree: {(double)dtCorrect / y_test.Length:P1}"); + +// RandomForest returns double[,] with columns: lower(0), median(1), upper(2), mean(3) +double[,] rfPredictions = randomForest.Predict(X_test); +int rfCorrect = 0; +for (int i = 0; i < y_test.Length; i++) + if (rfPredictions[i, 1] == y_test[i]) rfCorrect++; // Use median (column 1) +Console.WriteLine($" Random Forest: {(double)rfCorrect / y_test.Length:P1}"); + +// KNN returns double[] +double[] knnPredictions = knn.Predict(X_test); +int knnCorrect = 0; +for (int i = 0; i < y_test.Length; i++) + if (knnPredictions[i] == y_test[i]) knnCorrect++; +Console.WriteLine($" KNN (k=3): {(double)knnCorrect / y_test.Length:P1}"); +``` + +## Best Practices + +### Supervised Learning +1. **Split data** - Use train/test split or cross-validation +2. **Normalize features** - Especially for distance-based methods (KNN) +3. **Handle imbalanced classes** - Use stratified sampling or class weights +4. **Tune hyperparameters** - Grid search or random search +5. **Validate assumptions** - Check residuals for GLM +6. **Ensemble methods** - Random Forests often outperform single trees + +### Unsupervised Learning +1. **Scale features** - Clustering sensitive to feature scales +2. **Choose k carefully** - Use elbow method or silhouette scores +3. **Multiple runs** - k-Means sensitive to initialization +4. **Validate clusters** - Inspect cluster characteristics +5. **Consider GMM** - When clusters overlap or have different shapes + +--- + +## References + +[1] Nelder, J. A., & Wedderburn, R. W. M. (1972). Generalized linear models. *Journal of the Royal Statistical Society: Series A*, 135(3), 370-384. + +[2] Breiman, L., Friedman, J. H., Olshen, R. A., & Stone, C. J. (1984). *Classification and Regression Trees*. Wadsworth. + +[3] Breiman, L. (2001). Random forests. *Machine Learning*, 45(1), 5-32. + +[4] Cover, T., & Hart, P. (1967). Nearest neighbor pattern classification. *IEEE Transactions on Information Theory*, 13(1), 21-27. + +[5] Zhang, H. (2004). The optimality of naive Bayes. *Proceedings of the Seventeenth International FLAIRS Conference*, 562-567. + +[6] MacQueen, J. (1967). Some methods for classification and analysis of multivariate observations. *Proceedings of the Fifth Berkeley Symposium on Mathematical Statistics and Probability*, 1, 281-297. + +[7] Bishop, C. M. (2006). *Pattern Recognition and Machine Learning*. Springer. + +[8] Jenks, G. F. (1967). The data model concept in statistical mapping. *International Yearbook of Cartography*, 7, 186-190. + +[9] Hastie, T., Tibshirani, R., & Friedman, J. (2009). *The Elements of Statistical Learning* (2nd ed.). Springer. + +[10] Dempster, A. P., Laird, N. M., & Rubin, D. B. (1977). Maximum likelihood from incomplete data via the EM algorithm. *Journal of the Royal Statistical Society: Series B*, 39(1), 1-38. + +[11] Arthur, D., & Vassilvitskii, S. (2007). k-means++: The advantages of careful seeding. *Proceedings of the 18th Annual ACM-SIAM Symposium on Discrete Algorithms*, 1027-1035. + +--- + +[← Previous: Multivariate Distributions](../distributions/multivariate.md) | [Back to Index](../index.md) | [Next: Random Generation →](../sampling/random-generation.md) diff --git a/docs/machine-learning/overview.md b/docs/machine-learning/overview.md deleted file mode 100644 index 85624eb8..00000000 --- a/docs/machine-learning/overview.md +++ /dev/null @@ -1,711 +0,0 @@ -# Machine Learning - -[← Back to Index](../index.md) - -The ***Numerics*** library provides machine learning algorithms for both supervised and unsupervised learning tasks. These implementations are designed for engineering and scientific applications including classification, regression, and clustering. - -## Overview - -**Supervised Learning:** -- Generalized Linear Models (GLM) -- Decision Trees -- Random Forests -- k-Nearest Neighbors (KNN) -- Naive Bayes - -**Unsupervised Learning:** -- k-Means Clustering -- Gaussian Mixture Models (GMM) -- Jenks Natural Breaks - ---- - -## Supervised Learning - -### Generalized Linear Models (GLM) - -GLMs extend linear regression to non-normal response distributions [[1]](#1): - -```cs -using Numerics.MachineLearning; -using Numerics.Mathematics.LinearAlgebra; - -// Training data -double[,] X = { - { 1, 2.5, 1.2 }, // Observation 1: [intercept, feature1, feature2] - { 1, 3.1, 1.5 }, - { 1, 2.8, 1.1 }, - { 1, 3.5, 1.8 }, - { 1, 2.2, 0.9 } -}; - -double[] y = { 45.2, 52.3, 47.8, 58.1, 42.5 }; // Response variable - -// Create GLM -var glm = new GeneralizedLinearModel( - x: new Matrix(X), - y: new Vector(y), - family: GLMFamily.Normal, // Distribution family - linkFunction: LinkFunction.Identity // Link function -); - -// Set optimizer (optional) -glm.SetOptimizer(LocalMethod.NelderMead); - -// Train model -glm.Train(); - -Console.WriteLine("GLM Results:"); -Console.WriteLine($"Parameters: [{string.Join(", ", glm.Parameters.Select(p => p.ToString("F4")))}]"); -Console.WriteLine($"Standard Errors: [{string.Join(", ", glm.ParameterStandardErrors.Select(se => se.ToString("F4")))}]"); -Console.WriteLine($"p-values: [{string.Join(", ", glm.ParameterPValues.Select(p => p.ToString("F4")))}]"); - -// Model selection criteria -Console.WriteLine($"\nModel Selection:"); -Console.WriteLine($" AIC: {glm.AIC:F2}"); -Console.WriteLine($" BIC: {glm.BIC:F2}"); -Console.WriteLine($" Standard Error: {glm.StandardError:F4}"); - -// Make predictions -double[,] XNew = { - { 1, 3.0, 1.4 }, - { 1, 2.6, 1.0 } -}; - -double[] predictions = glm.Predict(new Matrix(XNew)); - -Console.WriteLine($"\nPredictions:"); -for (int i = 0; i < predictions.Length; i++) -{ - Console.WriteLine($" X_new[{i}] → {predictions[i]:F2}"); -} - -// Prediction intervals (alpha = 0.1 for 90% interval) -double[,] intervals = glm.Predict(new Matrix(XNew), alpha: 0.1); - -Console.WriteLine($"\n90% Prediction Intervals:"); -for (int i = 0; i < XNew.GetLength(0); i++) -{ - Console.WriteLine($" X_new[{i}]: [{intervals[i, 0]:F2}, {intervals[i, 1]:F2}]"); -} -``` - -**Supported Families:** -- `GLMFamily.Normal` - Gaussian (linear regression) -- `GLMFamily.Binomial` - Binary outcomes (logistic regression) -- `GLMFamily.Poisson` - Count data -- `GLMFamily.Gamma` - Positive continuous data - -**Link Functions:** -- `LinkFunction.Identity` - g(μ) = μ -- `LinkFunction.Log` - g(μ) = log(μ) -- `LinkFunction.Logit` - g(μ) = log(μ/(1-μ)) -- `LinkFunction.Probit` - g(μ) = Φ⁻¹(μ) - -### Decision Trees - -Classification and regression trees [[2]](#2): - -```cs -using Numerics.MachineLearning; - -// Classification example -double[,] X = { - { 5.1, 3.5, 1.4, 0.2 }, // Iris features - { 4.9, 3.0, 1.4, 0.2 }, - { 7.0, 3.2, 4.7, 1.4 }, - { 6.4, 3.2, 4.5, 1.5 }, - { 6.3, 3.3, 6.0, 2.5 }, - { 5.8, 2.7, 5.1, 1.9 } -}; - -double[] y = { 0, 0, 1, 1, 2, 2 }; // Classes: Setosa(0), Versicolor(1), Virginica(2) - -// Create decision tree -var tree = new DecisionTree( - X: X, - y: y, - maxDepth: 5, // Maximum tree depth - minSamplesSplit: 2, // Minimum samples to split node - minSamplesLeaf: 1 // Minimum samples in leaf -); - -// Train -tree.Train(); - -Console.WriteLine($"Decision Tree Trained: {tree.IsTrained}"); - -// Predict -double[] testSample = { 5.0, 3.0, 1.6, 0.2 }; -double[] prediction = tree.Predict(testSample); - -Console.WriteLine($"Prediction for test sample: Class {prediction[0]}"); - -// Predict multiple samples -double[,] testSamples = { - { 5.0, 3.0, 1.6, 0.2 }, - { 6.0, 3.0, 4.5, 1.5 }, - { 6.5, 3.0, 5.5, 2.0 } -}; - -double[] predictions = tree.Predict(testSamples); - -Console.WriteLine("\nBatch predictions:"); -for (int i = 0; i < predictions.Length; i++) -{ - Console.WriteLine($" Sample {i}: Class {predictions[i]}"); -} -``` - -### Random Forests - -Ensemble of decision trees for improved accuracy [[3]](#3): - -```cs -using Numerics.MachineLearning; - -double[,] X = { - // Same Iris data as above - { 5.1, 3.5, 1.4, 0.2 }, - { 4.9, 3.0, 1.4, 0.2 }, - { 7.0, 3.2, 4.7, 1.4 }, - { 6.4, 3.2, 4.5, 1.5 }, - { 6.3, 3.3, 6.0, 2.5 }, - { 5.8, 2.7, 5.1, 1.9 } -}; - -double[] y = { 0, 0, 1, 1, 2, 2 }; - -// Create random forest -var forest = new RandomForest( - X: X, - y: y, - nTrees: 100, // Number of trees - maxDepth: 5, - minSamplesSplit: 2, - minSamplesLeaf: 1, - maxFeatures: 2, // Features per split - bootstrap: true, // Bootstrap sampling - seed: 12345 -); - -// Train -forest.Train(); - -Console.WriteLine($"Random Forest Trained: {forest.IsTrained}"); -Console.WriteLine($"Number of trees: {forest.NTrees}"); - -// Predict with confidence intervals -double[] testSample = { 5.0, 3.0, 1.6, 0.2 }; -double[,] result = forest.Predict(testSample, alpha: 0.1); // 90% CI - -Console.WriteLine($"\nPrediction:"); -Console.WriteLine($" Predicted class: {result[0, 0]:F0}"); -Console.WriteLine($" 90% CI: [{result[0, 1]:F2}, {result[0, 2]:F2}]"); - -// Batch prediction -double[,] testSamples = { - { 5.0, 3.0, 1.6, 0.2 }, - { 6.0, 3.0, 4.5, 1.5 } -}; - -double[,] results = forest.Predict(testSamples, alpha: 0.1); - -Console.WriteLine($"\nBatch predictions:"); -for (int i = 0; i < testSamples.GetLength(0); i++) -{ - Console.WriteLine($" Sample {i}: Class {results[i, 0]:F0}, " + - $"CI [{results[i, 1]:F2}, {results[i, 2]:F2}]"); -} -``` - -**Advantages of Random Forests:** -- Reduces overfitting compared to single tree -- Provides prediction uncertainty -- Handles missing values well -- Works with mixed feature types - -### k-Nearest Neighbors (KNN) - -Non-parametric classification and regression [[4]](#4): - -```cs -using Numerics.MachineLearning; - -double[,] X = { - { 1.0, 2.0 }, - { 1.5, 1.8 }, - { 5.0, 8.0 }, - { 8.0, 8.0 }, - { 1.0, 0.6 }, - { 9.0, 11.0 } -}; - -double[] y = { 0, 0, 1, 1, 0, 1 }; // Binary classification - -// Create KNN classifier -var knn = new KNearestNeighbors( - X: X, - y: y, - k: 3, // Number of neighbors - weights: "uniform" // "uniform" or "distance" -); - -// KNN doesn't require explicit training -// Prediction happens at query time - -// Predict -double[] testPoint = { 2.0, 3.0 }; -double prediction = knn.Predict(testPoint); - -Console.WriteLine($"KNN Prediction for [{testPoint[0]}, {testPoint[1]}]: Class {prediction}"); - -// Predict with probability estimates -double[,] probs = knn.PredictProba(testPoint); - -Console.WriteLine($"Class probabilities:"); -for (int i = 0; i < probs.GetLength(0); i++) -{ - Console.WriteLine($" Class {i}: {probs[i, 0]:P1}"); -} -``` - -**Distance Metrics:** -- Euclidean (default) -- Manhattan -- Minkowski - -**Choosing k:** -- Small k: More sensitive to noise -- Large k: Smoother boundaries -- Rule of thumb: k = √n or use cross-validation - -### Naive Bayes - -Probabilistic classifier based on Bayes' theorem [[5]](#5): - -```cs -using Numerics.MachineLearning; - -// Text classification example (word counts) -double[,] X = { - { 2, 1, 0, 1 }, // Document 1: word counts - { 1, 1, 1, 0 }, - { 0, 3, 2, 1 }, - { 1, 0, 1, 2 } -}; - -double[] y = { 0, 0, 1, 1 }; // Classes: spam(1), ham(0) - -// Create Naive Bayes -var nb = new NaiveBayes(X: X, y: y); - -// Train -nb.Train(); - -Console.WriteLine("Naive Bayes trained"); - -// Predict -double[] testDoc = { 1, 2, 0, 1 }; -double prediction = nb.Predict(testDoc); - -Console.WriteLine($"Prediction: Class {prediction}"); - -// Class probabilities -double[] probabilities = nb.PredictProba(testDoc); - -Console.WriteLine($"Class probabilities:"); -Console.WriteLine($" Class 0 (ham): {probabilities[0]:P1}"); -Console.WriteLine($" Class 1 (spam): {probabilities[1]:P1}"); -``` - -**Assumptions:** -- Features are conditionally independent given class -- Works well despite violation of independence -- Fast training and prediction -- Good for text classification - ---- - -## Unsupervised Learning - -### k-Means Clustering - -Partition data into k clusters [[6]](#6): - -```cs -using Numerics.MachineLearning; - -// 2D data points -double[,] X = { - { 1.0, 2.0 }, - { 1.5, 1.8 }, - { 5.0, 8.0 }, - { 8.0, 8.0 }, - { 1.0, 0.6 }, - { 9.0, 11.0 }, - { 8.0, 2.0 }, - { 10.0, 2.0 }, - { 9.0, 3.0 } -}; - -// Create k-means with 3 clusters -var kmeans = new KMeans(X: X, k: 3); - -// Configure -kmeans.MaxIterations = 100; -kmeans.Tolerance = 1e-4; -kmeans.Seed = 12345; - -// Fit -kmeans.Fit(); - -Console.WriteLine($"k-Means Clustering (k={kmeans.K}):"); -Console.WriteLine($"Converged: {kmeans.HasConverged}"); -Console.WriteLine($"Iterations: {kmeans.Iterations}"); -Console.WriteLine($"Inertia: {kmeans.Inertia:F2}"); - -// Cluster centers -Console.WriteLine($"\nCluster Centers:"); -for (int i = 0; i < kmeans.K; i++) -{ - Console.WriteLine($" Cluster {i}: [{kmeans.Means[i, 0]:F2}, {kmeans.Means[i, 1]:F2}]"); -} - -// Cluster labels -Console.WriteLine($"\nCluster Assignments:"); -for (int i = 0; i < X.GetLength(0); i++) -{ - Console.WriteLine($" Point [{X[i, 0]:F1}, {X[i, 1]:F1}] → Cluster {kmeans.Labels[i]}"); -} - -// Predict cluster for new point -double[] newPoint = { 2.0, 3.0 }; -int cluster = kmeans.Predict(newPoint); - -Console.WriteLine($"\nNew point [{newPoint[0]}, {newPoint[1]}] → Cluster {cluster}"); - -// Cluster sizes -var clusterSizes = kmeans.Labels.GroupBy(l => l).Select(g => g.Count()).ToArray(); -Console.WriteLine($"\nCluster sizes: [{string.Join(", ", clusterSizes)}]"); -``` - -**Choosing k:** -- Elbow method (plot inertia vs. k) -- Silhouette analysis -- Domain knowledge - -**Initialization Methods:** -- Random selection -- k-means++ (default, better initialization) - -### Gaussian Mixture Models (GMM) - -Probabilistic clustering with soft assignments [[7]](#7): - -```cs -using Numerics.MachineLearning; - -double[,] X = { - // Same data as k-means example - { 1.0, 2.0 }, { 1.5, 1.8 }, { 5.0, 8.0 }, - { 8.0, 8.0 }, { 1.0, 0.6 }, { 9.0, 11.0 } -}; - -// Create GMM with 2 components -var gmm = new GaussianMixtureModel( - X: X, - nComponents: 2, - covarianceType: "full" // "full", "tied", "diag", "spherical" -); - -// Configure -gmm.MaxIterations = 100; -gmm.Tolerance = 1e-3; -gmm.Seed = 12345; - -// Fit using EM algorithm -gmm.Fit(); - -Console.WriteLine($"GMM Clustering ({gmm.NComponents} components):"); -Console.WriteLine($"Converged: {gmm.HasConverged}"); -Console.WriteLine($"Log-likelihood: {gmm.LogLikelihood:F2}"); -Console.WriteLine($"BIC: {gmm.BIC:F2}"); -Console.WriteLine($"AIC: {gmm.AIC:F2}"); - -// Component parameters -Console.WriteLine($"\nComponent Parameters:"); -for (int i = 0; i < gmm.NComponents; i++) -{ - Console.WriteLine($" Component {i}:"); - Console.WriteLine($" Weight: {gmm.Weights[i]:F3}"); - Console.WriteLine($" Mean: [{string.Join(", ", gmm.Means[i].Select(m => m.ToString("F2")))}]"); -} - -// Predict (hard assignment) -double[] newPoint = { 2.0, 3.0 }; -int component = gmm.Predict(newPoint); - -Console.WriteLine($"\nNew point [{newPoint[0]}, {newPoint[1]}] → Component {component}"); - -// Predict probabilities (soft assignment) -double[] probabilities = gmm.PredictProba(newPoint); - -Console.WriteLine($"Component probabilities:"); -for (int i = 0; i < probabilities.Length; i++) -{ - Console.WriteLine($" Component {i}: {probabilities[i]:P1}"); -} -``` - -**Advantages over k-Means:** -- Soft clustering (probabilistic assignments) -- Flexible cluster shapes (elliptical vs. spherical) -- Provides uncertainty quantification -- Can model overlapping clusters - -### Jenks Natural Breaks - -Optimal classification for univariate data [[8]](#8): - -```cs -using Numerics.MachineLearning; - -// Data values (e.g., elevation, rainfall, etc.) -double[] data = { 10, 12, 15, 18, 22, 25, 28, 35, 40, 45, 50, 55, 60, 70, 80 }; - -// Find natural breaks with 4 classes -int nClasses = 4; -var jenks = new JenksNaturalBreaks(data, nClasses); - -jenks.Compute(); - -Console.WriteLine($"Jenks Natural Breaks ({nClasses} classes):"); -Console.WriteLine($"Class breaks: [{string.Join(", ", jenks.Breaks.Select(b => b.ToString("F1")))}]"); -Console.WriteLine($"Goodness of variance fit: {jenks.GoodnessOfVarianceFit:F4}"); - -// Classify data -int[] classes = jenks.Classify(data); - -Console.WriteLine($"\nData classification:"); -for (int i = 0; i < Math.Min(10, data.Length); i++) -{ - Console.WriteLine($" Value {data[i]:F1} → Class {classes[i]}"); -} - -// Class statistics -for (int c = 0; c < nClasses; c++) -{ - var classData = data.Where((v, i) => classes[i] == c).ToArray(); - Console.WriteLine($"\nClass {c}:"); - Console.WriteLine($" Range: [{classData.Min():F1}, {classData.Max():F1}]"); - Console.WriteLine($" Count: {classData.Length}"); - Console.WriteLine($" Mean: {classData.Average():F1}"); -} -``` - -**Applications:** -- Choropleth map classification -- Data binning for visualization -- Natural grouping identification -- Minimizes within-class variance - ---- - -## Practical Examples - -### Example 1: Regression with GLM - -```cs -using Numerics.MachineLearning; -using Numerics.Mathematics.LinearAlgebra; - -// Predict home prices -double[,] features = { - { 1, 1500, 3, 20 }, // [intercept, sqft, bedrooms, age] - { 1, 1800, 4, 15 }, - { 1, 1200, 2, 30 }, - { 1, 2000, 4, 10 }, - { 1, 1600, 3, 25 } -}; - -double[] prices = { 250000, 320000, 190000, 380000, 270000 }; // $ - -var glm = new GeneralizedLinearModel( - new Matrix(features), - new Vector(prices), - GLMFamily.Normal, - LinkFunction.Identity -); - -glm.Train(); - -Console.WriteLine("Home Price Prediction Model:"); -Console.WriteLine($"Coefficients:"); -Console.WriteLine($" Intercept: ${glm.Parameters[0]:F0}"); -Console.WriteLine($" Per sqft: ${glm.Parameters[1]:F2}"); -Console.WriteLine($" Per bedroom: ${glm.Parameters[2]:F0}"); -Console.WriteLine($" Per year age: ${glm.Parameters[3]:F0}"); - -// Predict new home -double[,] newHome = { { 1, 1700, 3, 12 } }; -double predicted = glm.Predict(new Matrix(newHome))[0]; -double[,] interval = glm.Predict(new Matrix(newHome), alpha: 0.1); - -Console.WriteLine($"\nPrediction for 1700 sqft, 3BR, 12 years:"); -Console.WriteLine($" Predicted price: ${predicted:F0}"); -Console.WriteLine($" 90% Interval: [${interval[0, 0]:F0}, ${interval[0, 1]:F0}]"); -``` - -### Example 2: Classification Pipeline - -```cs -// Iris classification -double[,] X_train = LoadIrisFeatures(); // Load training data -double[] y_train = LoadIrisLabels(); -double[,] X_test = LoadIrisTestFeatures(); -double[] y_test = LoadIrisTestLabels(); - -// Train random forest -var rf = new RandomForest(X_train, y_train, nTrees: 100, seed: 42); -rf.Train(); - -// Evaluate -double[,] predictions = rf.Predict(X_test); -int correct = 0; -for (int i = 0; i < y_test.Length; i++) -{ - if (predictions[i, 0] == y_test[i]) - correct++; -} - -double accuracy = (double)correct / y_test.Length; - -Console.WriteLine($"Random Forest Classification:"); -Console.WriteLine($" Accuracy: {accuracy:P1}"); -Console.WriteLine($" Correct: {correct}/{y_test.Length}"); -``` - -### Example 3: Customer Segmentation - -```cs -// Customer data: [annual_spending, visit_frequency, avg_basket_size] -double[,] customers = { - { 1200, 24, 50 }, // Regular customer - { 5000, 52, 95 }, // High-value customer - { 300, 6, 45 }, // Occasional customer - { 4800, 48, 100 }, // High-value customer - { 800, 12, 65 }, // Regular customer - { 250, 4, 55 }, // Occasional customer - { 6000, 60, 105 } // VIP customer -}; - -// Cluster into 3 segments -var kmeans = new KMeans(customers, k: 3); -kmeans.Fit(); - -Console.WriteLine("Customer Segmentation:"); -for (int i = 0; i < 3; i++) -{ - var segment = Enumerable.Range(0, customers.GetLength(0)) - .Where(j => kmeans.Labels[j] == i) - .ToArray(); - - Console.WriteLine($"\nSegment {i} ({segment.Length} customers):"); - Console.WriteLine($" Avg spending: ${segment.Average(j => customers[j, 0]):F0}"); - Console.WriteLine($" Avg visits: {segment.Average(j => customers[j, 1]):F0}/year"); - Console.WriteLine($" Avg basket: ${segment.Average(j => customers[j, 2]):F0}"); -} -``` - -## Model Selection and Evaluation - -### Cross-Validation - -```cs -// Simple k-fold cross-validation -int k = 5; -int n = X.GetLength(0); -int foldSize = n / k; - -double[] accuracies = new double[k]; - -for (int fold = 0; fold < k; fold++) -{ - // Split data into train/test - var trainIndices = Enumerable.Range(0, n) - .Where(i => i < fold * foldSize || i >= (fold + 1) * foldSize) - .ToArray(); - - var testIndices = Enumerable.Range(fold * foldSize, foldSize).ToArray(); - - // Train and evaluate - // ... (extract train/test sets, train model, compute accuracy) - - accuracies[fold] = ComputeAccuracy(testIndices); -} - -Console.WriteLine($"Cross-Validation Results:"); -Console.WriteLine($" Mean accuracy: {accuracies.Average():P1}"); -Console.WriteLine($" Std dev: {Statistics.StandardDeviation(accuracies):F4}"); -``` - -### Model Comparison - -```cs -// Compare models on same dataset -var models = new[] { - ("Decision Tree", new DecisionTree(X, y)), - ("Random Forest", new RandomForest(X, y, nTrees: 50)), - ("KNN (k=3)", new KNearestNeighbors(X, y, k: 3)) -}; - -Console.WriteLine("Model Comparison:"); -foreach (var (name, model) in models) -{ - model.Train(); - double accuracy = EvaluateModel(model, X_test, y_test); - Console.WriteLine($" {name}: {accuracy:P1}"); -} -``` - -## Best Practices - -### Supervised Learning -1. **Split data** - Use train/test split or cross-validation -2. **Normalize features** - Especially for distance-based methods (KNN) -3. **Handle imbalanced classes** - Use stratified sampling or class weights -4. **Tune hyperparameters** - Grid search or random search -5. **Validate assumptions** - Check residuals for GLM -6. **Ensemble methods** - Random Forests often outperform single trees - -### Unsupervised Learning -1. **Scale features** - Clustering sensitive to feature scales -2. **Choose k carefully** - Use elbow method or silhouette scores -3. **Multiple runs** - k-Means sensitive to initialization -4. **Validate clusters** - Inspect cluster characteristics -5. **Consider GMM** - When clusters overlap or have different shapes - ---- - -## References - -[1] Nelder, J. A., & Wedderburn, R. W. M. (1972). Generalized linear models. *Journal of the Royal Statistical Society: Series A*, 135(3), 370-384. - -[2] Breiman, L., Friedman, J., Stone, C. J., & Olshen, R. A. (1984). *Classification and Regression Trees*. CRC Press. - -[3] Breiman, L. (2001). Random forests. *Machine Learning*, 45(1), 5-32. - -[4] Cover, T., & Hart, P. (1967). Nearest neighbor pattern classification. *IEEE Transactions on Information Theory*, 13(1), 21-27. - -[5] Zhang, H. (2004). The optimality of naive Bayes. *AA*, 1(2), 3. - -[6] MacQueen, J. (1967). Some methods for classification and analysis of multivariate observations. *Proceedings of the Fifth Berkeley Symposium on Mathematical Statistics and Probability*, 1(14), 281-297. - -[7] Bishop, C. M. (2006). *Pattern Recognition and Machine Learning*. Springer. - -[8] Jenks, G. F. (1967). The data model concept in statistical mapping. *International Yearbook of Cartography*, 7, 186-190. - ---- - -[← Back to Index](../index.md) diff --git a/docs/mathematics/differentiation.md b/docs/mathematics/differentiation.md index ee13c545..95c63c46 100644 --- a/docs/mathematics/differentiation.md +++ b/docs/mathematics/differentiation.md @@ -15,16 +15,10 @@ In ***Numerics***, the derivative is evaluated using the two-point (central diff where $x$ is the input point and $h$ represents a small change in $x$. In ***Numerics***, the step size $h$ is automatically determined according to the magnitude of $x$: ```math -\begin{equation} - h = - \begin{cases} - \mid x \mid \cdot \epsilon^\frac{1}{2} & x \neq 0\\ - \epsilon^\frac{1}{2} & x = 0\\ - \end{cases} -\end{equation} +h = \epsilon^{1/2} \cdot (1 + |x|) ``` -where $\epsilon$ is double precision machine epsilon. The step size $h$ can also be user-defined. +where $\epsilon$ is double precision machine epsilon. The $(1 + |x|)$ term scales the step size relative to the magnitude of the input, ensuring that the relative step size is appropriate regardless of the scale of $x$. The step size $h$ can also be user-defined. For example, consider the simple function: @@ -406,10 +400,10 @@ double h2 = NumericalDerivative.CalculateStepSize(x: 2.0, order: 2); The step size is calculated as: ```math -h = |x| \cdot \epsilon^{1/(1+\text{order})} +h = \epsilon^{1/(1+\text{order})} \cdot (1 + |x|) ``` -where $\epsilon$ is machine epsilon. For $x=0$, the formula simplifies to $h = \epsilon^{1/(1+\text{order})}$. +where $\epsilon$ is machine epsilon. The $(1 + |x|)$ term scales the step size relative to the magnitude of the input, ensuring an appropriate relative step size regardless of the scale of $x$. ## Best Practices @@ -427,7 +421,7 @@ where $\epsilon$ is machine epsilon. For $x=0$, the formula simplifies to $h = \ ## Accuracy Considerations -The central difference formula has truncation error $O(h^2)$ and roundoff error $O(\epsilon/h)$, where $\epsilon$ is machine epsilon. The optimal step size balances these errors at approximately $h \approx \epsilon^{1/3}$ for first derivatives and $h \approx \epsilon^{1/4}$ for second derivatives. The automatic step sizing in ***Numerics*** uses $h \approx \epsilon^{1/2}$, which is a conservative choice that works well in practice. +The central difference formula has truncation error $O(h^2)$ and roundoff error $O(\epsilon/h)$, where $\epsilon$ is machine epsilon. The optimal step size balances these errors at approximately $h \approx \epsilon^{1/3}$ for first derivatives and $h \approx \epsilon^{1/4}$ for second derivatives. The automatic step sizing in ***Numerics*** uses the formula $h = \epsilon^{1/(1+\text{order})} \cdot (1 + |x|)$, which gives $h \approx \epsilon^{1/2}$ for first derivatives and $h \approx \epsilon^{1/3}$ for second derivatives. For the second derivative, the truncation error is $O(h^2)$ and roundoff error is $O(\epsilon/h^2)$, making it more sensitive to numerical noise than first derivatives. @@ -439,7 +433,7 @@ The numerical differentiation methods implemented in ***Numerics*** are based on [1] W. H. Press, S. A. Teukolsky, W. T. Vetterling and B. P. Flannery, *Numerical Recipes: The Art of Scientific Computing*, 3rd ed., Cambridge, UK: Cambridge University Press, 2007. -[2] C. J. F. Ridders, "Accurate computation of F'(x) and F'(x) F''(x)," *Advances in Engineering Software*, vol. 4, no. 2, pp. 75-76, 1982. +[2] C. J. F. Ridders, "Accurate computation of F'(x) and F'(x)F''(x)," *Advances in Engineering Software*, vol. 4, no. 2, pp. 75-76, 1982. --- diff --git a/docs/mathematics/integration.md b/docs/mathematics/integration.md index 19a82dca..08b502cd 100644 --- a/docs/mathematics/integration.md +++ b/docs/mathematics/integration.md @@ -1,6 +1,6 @@ # Numerical Integration -[← Back to Index](../index.md) | [Next: Numerical Differentiation →](differentiation.md) +[← Previous: Getting Started](../getting-started.md) | [Back to Index](../index.md) | [Next: Numerical Differentiation →](differentiation.md) Numerical integration, also known as numerical quadrature, is a fundamental technique for approximating definite integrals. It has wide-ranging applications in various scientific and engineering fields. For example, in statistics, the expected value of a random variable is calculated using an integral, and numerical integration can be employed to approximate this expected value. Many problems in engineering and physics cannot be solved analytically and must rely on numerical methods to approximate solutions. @@ -56,7 +56,7 @@ public double FX(double x) ### Trapezoidal Rule -The _Integration_ class is a static class that contains the Midpoint Rule, Trapezoidal Rule, Simpson's Rule, and the 10-point Gauss-Legendre integration methods. Let's first compute the integral using the Trapezoidal Rule with 10 bins (or steps): +The _Integration_ class is a sealed class that contains the Midpoint Rule, Trapezoidal Rule, Simpson's Rule, and the 10-point and 20-point Gauss-Legendre integration methods. Let's first compute the integral using the Trapezoidal Rule with 10 bins (or steps): ```cs double result = Integration.TrapezoidalRule(FX, 0, 1, 10); // 0.25249999999999995 @@ -73,9 +73,9 @@ We can see that this is much more precise. Alternatively, the ***Numerics*** library provides a _TrapezoidalRule_ class that extends the basic static method and provides additional functionality for computing integration error estimates: ```cs -var trap = new TrapezoidalRule(FX, 0, 1, intervals: 100); +var trap = new TrapezoidalRule(FX, 0, 1); trap.Integrate(); -double result = trap.Result; // 0.25024999999999995 +double result = trap.Result; Console.WriteLine($"Function evaluations: {trap.FunctionEvaluations}"); ``` @@ -96,12 +96,12 @@ double result = Integration.SimpsonsRule(FX, 0, 1, 10); // 0.25 Or using the class-based approach: ```cs -var simpson = new SimpsonsRule(FX, 0, 1, intervals: 100); +var simpson = new SimpsonsRule(FX, 0, 1); simpson.Integrate(); -double result = simpson.Result; // 0.25 +double result = simpson.Result; ``` -**Error**: Simpson's Rule is fourth-order accurate, $O(h^4)$, making it significantly more accurate than the Trapezoidal Rule for smooth functions. +**Accuracy**: Simpson's Rule is fourth-order accurate, $O(h^4)$, making it significantly more accurate than the Trapezoidal Rule for smooth functions. ### Gauss-Legendre Quadrature @@ -111,13 +111,21 @@ The Gauss-Legendre method uses optimal polynomial quadrature for smooth function \int_{-1}^{1} f(x)\,dx \approx \sum_{i=1}^{n} w_i f(x_i) ``` -where $x_i$ are roots of Legendre polynomials and $w_i$ are corresponding weights. The ***Numerics*** library provides a 10-point Gauss-Legendre method: +where $x_i$ are roots of Legendre polynomials and $w_i$ are corresponding weights. The ***Numerics*** library provides both a 10-point and a 20-point Gauss-Legendre method: ```cs double result = Integration.GaussLegendre(FX, 0, 1); // 0.25 ``` -**Error**: The 10-point Gauss-Legendre method is exact for polynomials of degree 19 or less. +**Accuracy**: The 10-point Gauss-Legendre method is exact for polynomials of degree 19 or less. + +For higher accuracy with non-polynomial smooth integrands, use the 20-point variant: + +```cs +double result = Integration.GaussLegendre20(FX, 0, 1); // 0.25 +``` + +**Accuracy**: The 20-point Gauss-Legendre method is exact for polynomials of degree 39 or less. ### Midpoint Rule @@ -191,7 +199,8 @@ var gk = new AdaptiveGaussKronrod(FX, 0, 1); gk.RelativeTolerance = 1e-12; gk.Integrate(); double result = gk.Result; -Console.WriteLine($"Estimated error: {gk.Status}"); +Console.WriteLine($"Status: {gk.Status}"); +Console.WriteLine($"Standard error: {gk.StandardError}"); ``` **Advantages**: Efficient error estimation, reuses function evaluations from the Gauss rule in the Kronrod extension. @@ -335,7 +344,7 @@ Console.WriteLine($"Result: {result:F6}"); Console.WriteLine($"Function evaluations: {mc.FunctionEvaluations}"); ``` -With 100,000 samples, we see that the result is close but still has a noticeable error. Now, let's run it again with the default setting, where the maximum iterations are $N=100,000,000$: +With 100,000 samples, we see that the result is close but still has a noticeable error. Now, let's run it again with the default setting, where the maximum iterations are $N=10,000,000$: ```cs var mc = new MonteCarloIntegration(PI, 2, a, b); @@ -349,7 +358,7 @@ This result is much closer to the true value of $\pi$. Unlike traditional methods, the complexity of Monte Carlo integration grows slowly with the number of dimensions, making it particularly useful for high-dimensional problems. The Monte Carlo approach is simple to implement in higher dimensions and can handle irregular domains and complex integrands. However, it converges slowly; the error decreases as $O \left( \frac{1}{\sqrt{N}} \right)$, meaning to halve the error, you need to quadruple the number of samples. -**Error**: $O(1/\sqrt{N})$ - independent of dimension. +**Convergence Rate**: $O(1/\sqrt{N})$ - independent of dimension. ### MISER (Recursive Stratified Sampling) @@ -555,7 +564,7 @@ All integration classes (except the static `Integration` methods) inherit from t - `Result`: The computed integral value - `Iterations`: Number of iterations performed - `FunctionEvaluations`: Number of function evaluations performed -- `Status`: Integration status (Success, Failure, MaxIterationsReached, etc.) +- `Status`: Integration status (Success, Failure, MaximumIterationsReached, etc.) Example of using these properties: @@ -583,7 +592,7 @@ The VEGAS integrator includes a special method for rare event analysis: ```cs var vegas = new Vegas(myFunction, dimensions, min, max); -vegas.ConfigureForRareEvents(); // Optimizes settings for rare event detection +vegas.ConfigureForRareEvents(1e-6); // Optimizes settings for rare event detection vegas.Integrate(); ``` @@ -605,4 +614,4 @@ This method adjusts the internal parameters to better handle integrands with ver --- -[← Back to Index](../index.md) | [Next: Numerical Differentiation →](differentiation.md) +[← Previous: Getting Started](../getting-started.md) | [Back to Index](../index.md) | [Next: Numerical Differentiation →](differentiation.md) diff --git a/docs/mathematics/linear-algebra.md b/docs/mathematics/linear-algebra.md index 615844d5..ee2b7f20 100644 --- a/docs/mathematics/linear-algebra.md +++ b/docs/mathematics/linear-algebra.md @@ -1,6 +1,6 @@ # Linear Algebra -[← Back to Index](../index.md) | [Next: Special Functions →](special-functions.md) +[← Previous: Root Finding](root-finding.md) | [Back to Index](../index.md) | [Next: Special Functions →](special-functions.md) The ***Numerics*** library provides `Matrix` and `Vector` classes for linear algebra operations. These classes support common operations needed for numerical computing, optimization, and statistical analysis. @@ -43,9 +43,11 @@ var I = Matrix.Identity(3); // 3x3 identity // Zero matrix var zeros = new Matrix(3, 3); // Initialized to zeros by default -// Diagonal matrix -double[] diag = { 1, 2, 3 }; -var D = Matrix.Diagonal(diag); +// Diagonal matrix from a vector (static method creates a diagonal matrix) +var D = Matrix.Diagonal(new Vector(new[] { 1.0, 2.0, 3.0 })); + +// Extract diagonal elements from an existing matrix (instance method) +double[] diagElements = D.Diagonal(); // Returns [1, 2, 3] Console.WriteLine("Identity Matrix:"); Console.WriteLine(I.ToString()); @@ -201,12 +203,12 @@ var A = new Matrix(new double[,] { try { var Ainv = A.Inverse(); - + Console.WriteLine("A:"); Console.WriteLine(A.ToString()); Console.WriteLine("\nA^-1:"); Console.WriteLine(Ainv.ToString()); - + // Verify: A * A^-1 = I var I = A * Ainv; Console.WriteLine("\nA * A^-1 (should be I):"); @@ -227,17 +229,19 @@ var A = new Matrix(new double[,] { { 7, 8, 9 } }); -// Get row -Vector row1 = A.GetRow(1); // Second row: [4, 5, 6] +// Get row (returns double[]) +double[] row1 = A.Row(1); // Second row: [4, 5, 6] -// Get column -Vector col2 = A.GetColumn(2); // Third column: [3, 6, 9] +// Get column (returns double[]) +double[] col2 = A.Column(2); // Third column: [3, 6, 9] -// Set row -A.SetRow(0, new Vector(new[] { 10.0, 11.0, 12.0 })); +// Set row values using direct indexing +for (int j = 0; j < A.NumberOfColumns; j++) + A[0, j] = new[] { 10.0, 11.0, 12.0 }[j]; -// Set column -A.SetColumn(1, new Vector(new[] { 20.0, 21.0, 22.0 })); +// Set column values using direct indexing +for (int i = 0; i < A.NumberOfRows; i++) + A[i, 1] = new[] { 20.0, 21.0, 22.0 }[i]; Console.WriteLine("Modified matrix:"); Console.WriteLine(A.ToString()); @@ -256,8 +260,8 @@ var v1 = new Vector(new[] { 1.0, 2.0, 3.0 }); // Create with size var v2 = new Vector(5); // Length 5, initialized to zeros -// Copy constructor -var v3 = new Vector(v1); +// Clone a vector +var v3 = v1.Clone(); Console.WriteLine($"Vector v1: {v1.ToString()}"); Console.WriteLine($"Length: {v1.Length}"); @@ -271,7 +275,7 @@ Console.WriteLine($"Length: {v1.Length}"); var a = new Vector(new[] { 1.0, 2.0, 3.0 }); var b = new Vector(new[] { 4.0, 5.0, 6.0 }); -double dot = a.DotProduct(b); // 1*4 + 2*5 + 3*6 = 32 +double dot = Vector.DotProduct(a, b); // 1*4 + 2*5 + 3*6 = 32 Console.WriteLine($"a · b = {dot}"); ``` @@ -285,8 +289,8 @@ double norm = v.Norm(); // √(3² + 4²) = 5 Console.WriteLine($"||v|| = {norm}"); -// Unit vector -var u = v.Normalize(); // u = v / ||v|| +// Unit vector (normalize manually) +var u = v / v.Norm(); // u = v / ||v|| Console.WriteLine($"Unit vector: {u.ToString()}"); Console.WriteLine($"||u|| = {u.Norm():F10}"); // Should be 1.0 @@ -305,7 +309,7 @@ var v3 = v1 + v2; var v4 = v2 - v1; // Scalar multiplication -var v5 = v1 * 2.0; // or 2.0 * v1 +var v5 = v1 * 2.0; Console.WriteLine($"v1 + v2 = {v3.ToString()}"); Console.WriteLine($"v2 - v1 = {v4.ToString()}"); @@ -474,6 +478,499 @@ Console.WriteLine($"Manhattan distance: {manhattan:F1}"); - Inverse is computationally expensive - avoid when possible - For solving Ax=b, specialized solvers are more efficient than computing A^-1 +## Matrix Decompositions + +The library provides standard matrix decomposition algorithms for solving linear systems, computing determinants, and analyzing matrix structure. Matrix decompositions are the workhorses of numerical linear algebra -- rather than computing a matrix inverse directly (which is both expensive and numerically fragile), decompositions factor a matrix into structured components that can be used to solve systems, compute determinants, and analyze matrix properties efficiently and stably. + +### LU Decomposition + +#### Mathematical Background + +LU decomposition factors a square matrix $A$ into the product of a lower triangular matrix $L$ and an upper triangular matrix $U$. In practice, partial pivoting is applied to improve numerical stability, yielding the factorization: + +```math +PA = LU +``` + +where: + +- $P$ is a permutation matrix that records row interchanges, +- $L$ is a lower triangular matrix with ones on the diagonal (unit lower triangular), +- $U$ is an upper triangular matrix. + +**Why partial pivoting matters.** Without pivoting, small diagonal elements can appear during Gaussian elimination, causing division by near-zero values that amplify rounding errors catastrophically. Partial pivoting selects the largest available element in each column as the pivot, keeping the multipliers in $L$ bounded and ensuring numerical stability for most practical problems. + +**Solving linear systems.** Once the factorization $PA = LU$ is computed, solving $Ax = b$ is reduced to two triangular solves: + +```math +Ly = Pb \quad \text{(forward substitution)} +``` + +```math +Ux = y \quad \text{(back substitution)} +``` + +Each triangular solve costs only $O(n^2)$ operations. This makes LU decomposition especially efficient when solving multiple systems with the same coefficient matrix but different right-hand sides ($Ax = b_1, Ax = b_2, \ldots$), because the $O(n^3/3)$ factorization is performed only once. + +**Computational cost:** $O(n^3/3)$ for the factorization, plus $O(n^2)$ per solve. + +#### API Reference + +The `LUDecomposition` class takes a square matrix in its constructor and immediately performs the factorization using outer-product Gaussian elimination with partial pivoting. The original matrix is not modified; an internal copy is used. + +```cs +using Numerics.Mathematics.LinearAlgebra; + +var A = new Matrix(new double[,] { + { 2, 1, 1 }, + { 4, 3, 3 }, + { 8, 7, 9 } +}); + +var lu = new LUDecomposition(A); + +// Solve Ax = b +var b = new Vector(new double[] { 1, 1, 1 }); +Vector x = lu.Solve(b); + +// Compute determinant +double det = lu.Determinant(); +Console.WriteLine($"det(A) = {det:F4}"); + +// Compute inverse +Matrix Ainv = lu.InverseA(); +``` + +**Key members:** + +| Member | Type | Description | +|--------|------|-------------| +| `LU` | `Matrix` | The combined L and U factors stored in a single matrix | +| `A` | `Matrix` | A copy of the original input matrix | +| `Solve(Vector b)` | `Vector` | Solves $Ax = b$ using forward and back substitution | +| `Solve(Matrix B)` | `Matrix` | Solves $AX = B$ for multiple right-hand sides | +| `Determinant()` | `double` | Computes $\det(A)$ from the product of diagonal elements of $U$ | +| `InverseA()` | `Matrix` | Computes $A^{-1}$ by solving $AX = I$ | + +### QR Decomposition + +#### Mathematical Background + +QR decomposition factors an $m \times n$ matrix $A$ (with $m \geq n$) into the product of an orthogonal matrix and an upper triangular matrix: + +```math +A = QR +``` + +where: + +- $Q$ is an $m \times m$ orthogonal matrix, meaning $Q^T Q = I$ (its columns are orthonormal), +- $R$ is an $m \times n$ upper triangular matrix. + +The ***Numerics*** library computes the QR factorization using **Householder reflections**. At each step $k$, a Householder matrix $H_k = I - \beta v v^T$ is applied to zero out the subdiagonal entries of column $k$. The product $Q = H_1 H_2 \cdots H_n$ is accumulated explicitly as the full orthogonal factor. + +**Application to least squares.** QR decomposition is the preferred method for solving overdetermined least squares problems: + +```math +\min_x \| Ax - b \|_2 +``` + +Because $Q$ is orthogonal, the 2-norm is preserved under multiplication by $Q^T$, and the problem reduces to solving the upper triangular system: + +```math +Rx = Q^T b +``` + +This approach is significantly more numerically stable than forming and solving the normal equations $A^T A x = A^T b$ directly. The normal equations square the condition number of $A$ (i.e., $\kappa(A^T A) = \kappa(A)^2$), which can cause severe loss of accuracy for ill-conditioned problems. The QR approach avoids this squaring entirely. + +**Computational cost:** $O(2mn^2 - 2n^3/3)$ for an $m \times n$ matrix. + +#### API Reference + +The `QRDecomposition` class takes a general real $m \times n$ matrix in its constructor and computes the factorization using Householder reflections. + +```cs +var A = new Matrix(new double[,] { + { 1, 1 }, + { 1, 2 }, + { 1, 3 } +}); + +var qr = new QRDecomposition(A); + +// Access factors +Matrix Q = qr.Q; // Orthogonal matrix +Matrix R = qr.RMatrix; // Upper triangular + +// Solve overdetermined system (least squares) +var b = new Vector(new double[] { 1, 2, 2 }); +Vector x = qr.Solve(b); +``` + +**Key members:** + +| Member | Type | Description | +|--------|------|-------------| +| `Q` | `Matrix` | The $m \times m$ orthogonal matrix | +| `RMatrix` | `Matrix` | The $m \times n$ upper triangular matrix | +| `Solve(Vector b)` | `Vector` | Solves $Ax = b$ in the least squares sense via back substitution on $Rx = Q^T b$ | +| `Solve(Matrix B)` | `Matrix` | Solves $AX = B$ for multiple right-hand sides | + +### Cholesky Decomposition + +#### Mathematical Background + +Cholesky decomposition factors a **symmetric positive-definite** (SPD) matrix $A$ into the product of a lower triangular matrix and its transpose: + +```math +A = LL^T +``` + +where $L$ is a lower triangular matrix with strictly positive diagonal entries. + +A matrix $A$ is symmetric positive-definite if $A = A^T$ and $x^T A x > 0$ for all nonzero vectors $x$. Common examples of SPD matrices include covariance matrices, correlation matrices, Gram matrices ($X^T X$ for full-rank $X$), and stiffness matrices in structural analysis. + +**Computational advantage.** Because the factorization exploits symmetry, Cholesky requires approximately half the work of LU decomposition: + +```math +\text{Cost} \approx O(n^3/6) +``` + +This makes it the fastest general-purpose direct solver for SPD systems. + +**Solving linear systems.** As with LU, the solution of $Ax = b$ proceeds by two triangular solves: + +```math +Ly = b \quad \text{(forward substitution)} +``` + +```math +L^T x = y \quad \text{(back substitution)} +``` + +**Connection to multivariate Normal sampling.** Cholesky decomposition is essential for generating correlated random variables. If $\Sigma = LL^T$ is a covariance matrix and $z \sim N(0, I)$ is a vector of independent standard Normal samples, then: + +```math +x = \mu + Lz \quad \implies \quad x \sim N(\mu, \Sigma) +``` + +This transformation is used extensively in Monte Carlo simulation, Bayesian inference, and risk analysis. + +**Positive-definiteness diagnostic.** If the Cholesky decomposition fails (a diagonal element becomes zero or negative during factorization), the matrix is not positive-definite. This is a useful diagnostic for detecting ill-conditioned or indefinite covariance matrices. + +#### API Reference + +The `CholeskyDecomposition` class takes a symmetric positive-definite matrix in its constructor. If the matrix is not SPD, the constructor throws an exception. + +```cs +// Symmetric positive-definite matrix (e.g., covariance matrix) +var A = new Matrix(new double[,] { + { 4, 2 }, + { 2, 3 } +}); + +var chol = new CholeskyDecomposition(A); + +// Check if decomposition succeeded +Console.WriteLine($"Positive definite: {chol.IsPositiveDefinite}"); + +// Lower triangular factor +Matrix L = chol.L; + +// Solve Ax = b +var b = new Vector(new double[] { 1, 1 }); +Vector x = chol.Solve(b); + +// Compute log-determinant (numerically stable) +double logDet = chol.LogDeterminant(); +Console.WriteLine($"log|A| = {logDet:F4}"); +``` + +**Key members:** + +| Member | Type | Description | +|--------|------|-------------| +| `L` | `Matrix` | The lower triangular Cholesky factor | +| `A` | `Matrix` | A copy of the original input matrix | +| `IsPositiveDefinite` | `bool` | `true` if the decomposition succeeded (matrix is SPD) | +| `Solve(Vector b)` | `Vector` | Solves $Ax = b$ via forward and back substitution on $LL^T$ | +| `Forward(Vector b)` | `Vector` | Solves the forward substitution step $Ly = b$ | +| `Backward(Vector y)` | `Vector` | Solves the back substitution step $L^T x = y$ | +| `InverseA()` | `Matrix` | Computes $A^{-1}$ using the stored decomposition | +| `Determinant()` | `double` | Computes $\det(A) = (\prod L_{ii})^2$ | +| `LogDeterminant()` | `double` | Computes $\log \det(A) = 2 \sum \log L_{ii}$ (numerically stable for large matrices) | + +### Eigenvalue Decomposition + +#### Mathematical Background + +An eigenvalue $\lambda$ and corresponding eigenvector $v$ of a square matrix $A$ satisfy the fundamental equation: + +```math +Av = \lambda v +``` + +That is, the matrix $A$ acts on the eigenvector $v$ by simply scaling it. The eigenvalues are the roots of the **characteristic polynomial**: + +```math +\det(A - \lambda I) = 0 +``` + +For a real **symmetric** matrix $A$, the eigenvalues are all real and the eigenvectors are orthogonal. This yields the **spectral decomposition**: + +```math +A = Q \Lambda Q^T +``` + +where $Q$ is an orthogonal matrix whose columns are the eigenvectors and $\Lambda = \text{diag}(\lambda_1, \lambda_2, \ldots, \lambda_n)$ is a diagonal matrix of eigenvalues. + +The ***Numerics*** library computes this decomposition using the **Jacobi rotation method**, which is an iterative algorithm that applies a sequence of plane rotations to systematically zero out off-diagonal elements. Each rotation is chosen to eliminate the largest remaining off-diagonal entry. The method converges when all off-diagonal elements are smaller than a tolerance ($10^{-12}$). The Jacobi method is robust and highly accurate for small to medium-sized symmetric matrices. + +**Applications:** + +- **Principal Component Analysis (PCA):** The eigenvectors of a covariance matrix define the principal directions of variation in data. +- **Stability analysis:** In dynamical systems, the eigenvalues of the system matrix determine whether the system is stable (all eigenvalues have negative real parts), neutrally stable, or unstable. +- **Modal analysis:** In structural engineering, the eigenvalues of the stiffness-mass system correspond to natural frequencies of vibration. + +**Connection to condition number.** For symmetric matrices, the condition number can be computed directly from the eigenvalues: + +```math +\kappa(A) = \left| \frac{\lambda_{\max}}{\lambda_{\min}} \right| +``` + +A large condition number indicates that the matrix is nearly singular and that solutions to linear systems involving $A$ may be unreliable. + +#### API Reference + +The `EigenValueDecomposition` class accepts a **symmetric** matrix and computes all eigenvalues and eigenvectors using the Jacobi rotation method. The constructor validates that the input matrix is both square and symmetric. + +```cs +var A = new Matrix(new double[,] { + { 2, 1 }, + { 1, 3 } +}); + +var eigen = new EigenValueDecomposition(A); + +// Eigenvalues +Console.WriteLine("Eigenvalues:"); +for (int i = 0; i < eigen.EigenValues.Length; i++) + Console.WriteLine($" λ{i} = {eigen.EigenValues[i]:F4}"); + +// Eigenvectors (columns of EigenVectors matrix) +Console.WriteLine("Eigenvectors:"); +Matrix V = eigen.EigenVectors; +``` + +**Key members:** + +| Member | Type | Description | +|--------|------|-------------| +| `EigenValues` | `Vector` | The eigenvalues $\lambda_1, \ldots, \lambda_n$ | +| `EigenVectors` | `Matrix` | Orthogonal matrix whose columns are the corresponding eigenvectors | +| `A` | `Matrix` | A copy of the original input matrix | +| `EffectiveSampleSize()` | `double` | Returns the effective sample size based on Dutilleul's method, computed as $(\sum \lambda_i)^2 / \sum \lambda_i^2$ | + +### Singular Value Decomposition (SVD) + +#### Mathematical Background + +The Singular Value Decomposition is arguably the most important and versatile matrix factorization in numerical linear algebra. Every real $m \times n$ matrix $A$ (regardless of shape or rank) admits the decomposition: + +```math +A = U \Sigma V^T +``` + +where: + +- $U$ is an $m \times m$ orthogonal matrix whose columns are the **left singular vectors**, +- $\Sigma$ is an $m \times n$ diagonal matrix containing the **singular values** $\sigma_1 \geq \sigma_2 \geq \cdots \geq \sigma_{\min(m,n)} \geq 0$, arranged in non-increasing order, +- $V$ is an $n \times n$ orthogonal matrix whose columns are the **right singular vectors**. + +The singular values are always non-negative real numbers. They represent the "stretching factors" along the principal axes of the linear transformation defined by $A$. + +**Relationship to eigenvalues.** The singular values of $A$ are the square roots of the eigenvalues of $A^T A$ (or equivalently $A A^T$). The right singular vectors are the eigenvectors of $A^T A$, and the left singular vectors are the eigenvectors of $A A^T$. + +**Pseudoinverse.** The SVD provides a natural way to compute the Moore-Penrose pseudoinverse $A^+$. If $A = U \Sigma V^T$, then: + +```math +A^+ = V \Sigma^+ U^T +``` + +where $\Sigma^+$ is formed by taking the reciprocal of each nonzero singular value on the diagonal. Singular values below a threshold are treated as zero, providing a numerically stable inverse even for rank-deficient matrices. + +**Numerical rank.** The number of singular values above a threshold determines the numerical rank of the matrix. This is far more reliable than computing rank via Gaussian elimination, which can be sensitive to rounding errors. + +**Low-rank approximation.** The Eckart-Young theorem states that the best rank-$k$ approximation to $A$ (in both the 2-norm and Frobenius norm) is obtained by retaining only the $k$ largest singular values: + +```math +A_k = \sum_{i=1}^{k} \sigma_i \, u_i \, v_i^T +``` + +**Condition number.** The condition number of $A$ in the 2-norm is the ratio of the largest to smallest singular value: + +```math +\kappa(A) = \frac{\sigma_{\max}}{\sigma_{\min}} +``` + +**Applications:** + +- **Least squares (rank-deficient):** SVD provides the minimum-norm least squares solution even when $A$ is rank-deficient, where QR decomposition would fail due to a singular $R$ factor. +- **Principal Component Analysis:** The right singular vectors define the principal components; the singular values quantify the variance explained by each component. +- **Signal/noise separation:** In data analysis, large singular values correspond to signal and small singular values correspond to noise. +- **Dimensionality reduction:** Truncated SVD retains only the most important components of a dataset. + +#### API Reference + +The `SingularValueDecomposition` class takes a general real $m \times n$ matrix and computes the full SVD. The implementation uses Golub-Kahan bidiagonalization followed by implicit QR iteration, and the singular values are automatically sorted in decreasing order. + +```cs +using Numerics.Mathematics.LinearAlgebra; + +var A = new Matrix(new double[,] { + { 1, 2 }, + { 3, 4 }, + { 5, 6 } +}); + +var svd = new SingularValueDecomposition(A); + +// Singular values (sorted in decreasing order) +Console.WriteLine("Singular values:"); +for (int i = 0; i < svd.W.Length; i++) + Console.WriteLine($" σ{i} = {svd.W[i]:F6}"); + +// Left singular vectors (m x m) +Matrix U = svd.U; + +// Right singular vectors (n x n) +Matrix V = svd.V; + +// Condition number (reciprocal) +Console.WriteLine($"1/κ(A) = {svd.InverseCondition:E4}"); + +// Numerical rank +int rank = svd.Rank(); +Console.WriteLine($"Rank: {rank}"); + +// Solve Ax = b using the pseudoinverse +var b = new Vector(new double[] { 1, 2, 3 }); +Vector x = svd.Solve(b); +Console.WriteLine($"Least squares solution: {x.ToString()}"); +``` + +**Key members:** + +| Member | Type | Description | +|--------|------|-------------| +| `U` | `Matrix` | The $m \times n$ matrix of left singular vectors (column-orthogonal) | +| `V` | `Matrix` | The $n \times n$ orthogonal matrix of right singular vectors | +| `W` | `Vector` | The singular values $\sigma_1 \geq \sigma_2 \geq \cdots \geq 0$, stored as a vector | +| `A` | `Matrix` | A copy of the original input matrix | +| `Threshold` | `double` | Threshold below which singular values are treated as zero (default is based on machine precision) | +| `InverseCondition` | `double` | The reciprocal of the condition number: $\sigma_{\min} / \sigma_{\max}$ | +| `Rank(threshold)` | `int` | Number of singular values above the threshold | +| `Nullity(threshold)` | `int` | Number of singular values at or below the threshold | +| `Range(threshold)` | `Matrix` | Orthonormal basis for the column space of $A$ | +| `Nullspace(threshold)` | `Matrix` | Orthonormal basis for the null space of $A$ | +| `Solve(Vector b, threshold)` | `Vector` | Solves $Ax = b$ using the pseudoinverse | +| `Solve(Matrix B, threshold)` | `Matrix` | Solves $AX = B$ for multiple right-hand sides | +| `LogDeterminant()` | `double` | Computes $\sum \log \sigma_i$ | +| `LogPseudoDeterminant()` | `double` | Computes $\sum \log \sigma_i$ for nonzero $\sigma_i$ only | + +## Condition Number and Numerical Stability + +When solving linear systems or computing matrix operations, the **condition number** is the single most important diagnostic for assessing the reliability of your results. Understanding it is critical for any safety-critical application. + +### Definition + +The condition number of a matrix $A$ is defined as: + +```math +\kappa(A) = \|A\| \cdot \|A^{-1}\| +``` + +Using the 2-norm (spectral norm), this simplifies to the ratio of the largest to smallest singular value: + +```math +\kappa(A) = \frac{\sigma_{\max}}{\sigma_{\min}} +``` + +For symmetric matrices, this is equivalently the ratio of the largest to smallest eigenvalue in absolute value. + +### Interpretation + +The condition number quantifies how much errors in the input data (the matrix $A$ or the right-hand side $b$) are amplified in the solution. Specifically: + +- A relative perturbation of size $\epsilon$ in $A$ or $b$ can produce a relative error of up to $\kappa(A) \cdot \epsilon$ in the solution $x$. +- In floating-point arithmetic with approximately 16 digits of precision (double), a condition number of $10^k$ means you may **lose up to $k$ digits of accuracy** in the computed solution. + +### Practical Guidelines + +| Condition Number | Assessment | Action | +|-----------------|------------|--------| +| $\kappa \approx 1$ | Perfectly conditioned | Results are reliable | +| $\kappa \approx 10^3$ | Mildly ill-conditioned | Results are likely fine for most applications | +| $\kappa \approx 10^6$ | Moderately ill-conditioned | Results should be verified independently | +| $\kappa > 10^{10}$ | Severely ill-conditioned | Results are suspect; consider reformulating the problem | +| $\kappa \approx 10^{16}$ or $\sigma_{\min} \approx 0$ | Numerically singular | The matrix is effectively singular in double precision | + +### How to Check + +The most reliable way to compute the condition number is through the SVD: + +```cs +var A = new Matrix(new double[,] { + { 1, 2 }, + { 3, 4 } +}); + +var svd = new SingularValueDecomposition(A); + +// Reciprocal condition number (values near 0 indicate ill-conditioning) +double rcond = svd.InverseCondition; +Console.WriteLine($"1/κ(A) = {rcond:E4}"); + +// Full condition number +double cond = 1.0 / rcond; +Console.WriteLine($"κ(A) = {cond:F2}"); + +// Inspect individual singular values +Console.WriteLine("Singular values:"); +for (int i = 0; i < svd.W.Length; i++) + Console.WriteLine($" σ{i} = {svd.W[i]:E6}"); +``` + +**When to check.** Always check the condition number before trusting the results of a linear system solve, especially when: + +- The matrix comes from measured or uncertain data, +- The problem involves interpolation or extrapolation, +- The matrix is large and its structure is not well understood, +- Results are used in safety-critical decisions. + +## Choosing a Decomposition + +Selecting the right decomposition depends on the structure of your matrix and the problem you are solving. The following table provides a practical decision guide: + +| Problem | Matrix Properties | Recommended Decomposition | Rationale | +|---------|------------------|--------------------------|-----------| +| Solve $Ax = b$ (general) | Square, non-singular | **LU** | Fast $O(n^3/3)$ factorization; efficient for multiple right-hand sides | +| Solve $Ax = b$ (SPD) | Symmetric positive-definite | **Cholesky** | Half the cost of LU; exploits symmetry | +| Least squares $\min \|Ax - b\|_2$ | Overdetermined ($m > n$), full rank | **QR** | Numerically stable; avoids squaring the condition number | +| Least squares (rank-deficient) | Any shape, possibly rank-deficient | **SVD** | Provides minimum-norm solution even when $A$ is rank-deficient | +| Eigenvalues/eigenvectors | Symmetric | **Eigenvalue** | Spectral decomposition via Jacobi rotation | +| Condition number, numerical rank | Any | **SVD** | Singular values directly give $\kappa(A)$ and rank | +| Monte Carlo sampling | Covariance matrix (SPD) | **Cholesky** | $x = \mu + Lz$ generates correlated samples | +| Determinant | Square | **LU** or **Cholesky** | Product of diagonal elements (or their squares for Cholesky) | + +### Rules of Thumb + +1. **Start with LU** for general square systems. It is the standard workhorse. +2. **Use Cholesky** whenever you know the matrix is symmetric positive-definite. It is faster and the positive-definiteness check is a valuable diagnostic. +3. **Use QR** for least squares problems. Never solve normal equations with matrix inversion for production code -- it amplifies rounding errors. +4. **Use SVD** when you are unsure about the rank or conditioning of your matrix, or when you need the most robust solution possible. +5. **Use Eigenvalue decomposition** when you need the eigenvalues themselves (for spectral analysis, stability, PCA), not just to solve a linear system. + ## Common Operations Summary | Operation | Method | Complexity | @@ -483,8 +980,23 @@ Console.WriteLine($"Manhattan distance: {manhattan:F1}"); | Inverse | `A.Inverse()` | O(n³) | | Determinant | `A.Determinant()` | O(n³) | | Vector norm | `v.Norm()` | O(n) | -| Dot product | `v.DotProduct(w)` | O(n) | +| Dot product | `Vector.DotProduct(v, w)` | O(n) | +| LU factorization | `new LUDecomposition(A)` | O(n³/3) | +| QR factorization | `new QRDecomposition(A)` | O(2mn² - 2n³/3) | +| Cholesky factorization | `new CholeskyDecomposition(A)` | O(n³/6) | +| SVD | `new SingularValueDecomposition(A)` | O(min(mn², m²n)) | +| Eigenvalue (symmetric) | `new EigenValueDecomposition(A)` | O(n³) iterative | + +--- + +## References + +[1] Golub, G. H., & Van Loan, C. F. (2013). *Matrix Computations* (4th ed.). Johns Hopkins University Press. + +[2] Trefethen, L. N., & Bau, D. (1997). *Numerical Linear Algebra*. SIAM. + +[3] Press, W. H., Teukolsky, S. A., Vetterling, W. T., & Flannery, B. P. (2007). *Numerical Recipes: The Art of Scientific Computing* (3rd ed.). Cambridge University Press. --- -[← Back to Index](../index.md) | [Next: Special Functions →](special-functions.md) +[← Previous: Root Finding](root-finding.md) | [Back to Index](../index.md) | [Next: Special Functions →](special-functions.md) diff --git a/docs/mathematics/ode-solvers.md b/docs/mathematics/ode-solvers.md index 9a7b32a9..350ebf2a 100644 --- a/docs/mathematics/ode-solvers.md +++ b/docs/mathematics/ode-solvers.md @@ -1,28 +1,32 @@ # ODE Solvers -[← Previous: Special Functions](special-functions.md) | [Back to Index](../index.md) | [Next: Hypothesis Tests →](../statistics/hypothesis-tests.md) +[← Previous: Special Functions](special-functions.md) | [Back to Index](../index.md) | [Next: Interpolation →](../data/interpolation.md) -The ***Numerics*** library provides Runge-Kutta methods for solving Ordinary Differential Equations (ODEs). These methods are essential for modeling dynamic systems, population dynamics, chemical reactions, and physical processes. +An ordinary differential equation (ODE) relates a function to its derivatives. The general **initial value problem** has the form: -## Overview +```math +\frac{dy}{dt} = f(t, y), \quad y(t_0) = y_0 +``` -An ordinary differential equation has the form: +Given the initial condition $y(t_0) = y_0$, we seek the solution $y(t)$ for $t > t_0$. Most ODEs arising in practice — population dynamics, chemical kinetics, mechanical systems, heat transfer — cannot be solved analytically and require numerical methods. -``` -dy/dt = f(t, y) -``` +The ***Numerics*** library provides Runge-Kutta methods for solving ODEs numerically. These methods advance the solution from $y_n$ at time $t_n$ to $y_{n+1}$ at time $t_{n+1} = t_n + h$ by evaluating $f(t, y)$ at several intermediate points within the step. -Given initial condition y(t₀) = y₀, we want to find y(t) for t > t₀. +## Second-Order Runge-Kutta (RK2) -The `RungeKutta` class provides several methods for numerical integration: -- Second-order Runge-Kutta (RK2) -- Fourth-order Runge-Kutta (RK4) -- Runge-Kutta-Fehlberg (adaptive step size) -- Cash-Karp (adaptive step size) +The simplest Runge-Kutta method evaluates the derivative at the beginning and end of the step, then averages: -## Second-Order Runge-Kutta +```math +k_1 = f(t_n, y_n) +``` +```math +k_2 = f(t_n + h, y_n + h \cdot k_1) +``` +```math +y_{n+1} = y_n + \frac{h}{2}(k_1 + k_2) +``` -Simple but less accurate method: +This is equivalent to the explicit trapezoidal rule. It is second-order accurate, meaning the local truncation error per step is $O(h^3)$ and the global accumulated error is $O(h^2)$. ```cs using Numerics.Mathematics.ODESolvers; @@ -47,7 +51,40 @@ Console.WriteLine($"Error: {Math.Abs(solution[steps - 1] - Math.E):E4}"); ## Fourth-Order Runge-Kutta (RK4) -Most commonly used method - good balance of accuracy and efficiency: +The classical RK4 method is the workhorse of ODE solving — it provides an excellent balance of accuracy and efficiency. The method evaluates the derivative at four points within each step: + +```math +k_1 = f(t_n, y_n) +``` +```math +k_2 = f\!\left(t_n + \frac{h}{2}, \; y_n + \frac{h}{2} k_1\right) +``` +```math +k_3 = f\!\left(t_n + \frac{h}{2}, \; y_n + \frac{h}{2} k_2\right) +``` +```math +k_4 = f(t_n + h, \; y_n + h \cdot k_3) +``` +```math +y_{n+1} = y_n + \frac{h}{6}(k_1 + 2k_2 + 2k_3 + k_4) +``` + +The $k$-values can be interpreted as slope estimates: $k_1$ at the beginning, $k_2$ and $k_3$ at the midpoint (using different estimates to get there), and $k_4$ at the end. The weighted average gives $k_2$ and $k_3$ twice the weight of $k_1$ and $k_4$, similar to Simpson's rule in quadrature. + +### Butcher Tableau + +The coefficients of any Runge-Kutta method can be compactly represented in a **Butcher tableau**. For the classical RK4: + +``` + 0 | + 1/2 | 1/2 + 1/2 | 0 1/2 + 1 | 0 0 1 + ----|-------------------- + | 1/6 1/3 1/3 1/6 +``` + +The left column gives the time offsets $c_i$ (where the derivative is evaluated), the body of the table gives the coefficients $a_{ij}$ for combining previous $k$-values, and the bottom row gives the weights $b_i$ for the final combination. The local truncation error is $O(h^5)$ and the global error is $O(h^4)$, making it fourth-order accurate. ### Array Output (Multiple Time Steps) @@ -73,7 +110,7 @@ for (int i = 0; i < steps; i += 40) double t = t0 + i * dt; double numerical = solution[i]; double analytical = Math.Exp(-t * t); - + Console.WriteLine($"t={t:F2}: y_num={numerical:F6}, y_exact={analytical:F6}, " + $"error={Math.Abs(numerical - analytical):E4}"); } @@ -81,7 +118,7 @@ for (int i = 0; i < steps; i += 40) ### Single Step -Useful for step-by-step integration: +Useful for step-by-step integration or for building custom solvers: ```cs // Single RK4 step @@ -96,11 +133,40 @@ Console.WriteLine($"y({t + dt:F1}) = {y_next:F6}"); ## Adaptive Step Size Methods -Adaptive methods automatically adjust step size to maintain accuracy while minimizing computation. +Fixed-step methods require the user to choose $h$ in advance, but the optimal step size depends on the local behavior of the solution. If $h$ is too large, the solution may be inaccurate or unstable; if too small, computation is wasted. **Adaptive methods** adjust $h$ automatically by estimating the local error at each step. + +### Embedded Runge-Kutta Methods + +The key idea behind embedded methods is to compute **two solutions of different orders** using mostly the same function evaluations. The difference between the two solutions provides an error estimate without extra cost. + +Both the Fehlberg and Cash-Karp methods compute a fourth-order solution $y^{(4)}$ and a fifth-order solution $y^{(5)}$ using six stages ($k_1$ through $k_6$). The error estimate is: + +```math +\text{err} = |y^{(4)} - y^{(5)}| +``` + +The step size is then adjusted: +- If $\text{err} \leq \text{tolerance}$: **accept** the step. If the error is very small ($< 0.01 \times \text{tolerance}$), double the step size +- If $\text{err} > \text{tolerance}$: **reject** the step, halve $h$, and retry +- If $h$ falls below the minimum step size `dtMin`, accept the step regardless (to avoid infinite loops) ### Runge-Kutta-Fehlberg (RKF45) -Fifth-order accuracy with embedded fourth-order error estimate: +The Fehlberg method [[1]](#1) uses specific coefficients optimized to minimize the leading error term. Its Butcher tableau has the structure: + +``` + 0 | + 1/4 | 1/4 + 3/8 | 3/32 9/32 + 12/13 | 1932/2197 -7200/2197 7296/2197 + 1 | 439/216 -8 3680/513 -845/4104 + 1/2 | -8/27 2 -3544/2565 1859/4104 -11/40 + --------|------------------------------------------------------------ + RK4 | 25/216 0 1408/2565 2197/4104 -1/5 0 + RK5 | 16/135 0 6656/12825 28561/56430 -9/50 2/55 +``` + +The fourth-order solution (row labeled RK4) is used for stepping, while the difference from the fifth-order solution provides the error estimate. ```cs using Numerics.Mathematics.ODESolvers; @@ -122,7 +188,7 @@ Console.WriteLine($"Analytical: {Math.Exp(-15.0 * (t0 + dt)):F8}"); ### Cash-Karp Method -Alternative adaptive method with different error estimation: +The Cash-Karp method [[2]](#2) uses a different set of coefficients optimized for a different balance of accuracy and stability: ```cs double y_final_ck = RungeKutta.CashKarp(f, y0, t0, dt, dtMin, tolerance); @@ -131,10 +197,32 @@ Console.WriteLine($"Cash-Karp: y({t0 + dt:F3}) = {y_final_ck:F8}"); ``` **When to use adaptive methods:** -- Stiff equations -- Rapidly changing solutions -- Need to guarantee error tolerance -- Variable dynamics (slow then fast) +- When the solution dynamics change significantly over the integration interval (e.g., rapid transients followed by slow decay) +- When you need guaranteed error tolerance without manual tuning +- When the problem timescale is unknown in advance + +## Stiffness + +A differential equation is **stiff** when it contains dynamics on vastly different timescales — some components of the solution change rapidly while others change slowly. The classic example is a system with eigenvalues $\lambda_1 = -1$ and $\lambda_2 = -1000$: the fast component $e^{-1000t}$ decays in milliseconds, but the explicit RK4 step size is constrained by this fast component even long after it has decayed to zero. + +For the linear test equation $\frac{dy}{dt} = \lambda y$ (with $\lambda < 0$), the explicit RK4 method is stable only when: + +```math +|h \cdot \lambda| \leq 2.785 +``` + +This means that for $\lambda = -1000$, the step size must be smaller than $h < 0.002785$ — regardless of the accuracy requirement. For stiff problems, this stability restriction forces an impractically small step size. + +**Mitigation strategies:** +- Use the adaptive methods (Fehlberg, Cash-Karp), which will automatically reduce the step size in stiff regions +- For very stiff problems, consider implicit methods (not currently in the library) which have much larger stability regions +- If the stiff component has decayed, reformulate the problem to remove it + +## Error Accumulation + +The **local truncation error** is the error introduced in a single step assuming the previous value is exact. For RK4, this is $O(h^5)$ per step. The **global error** after integrating over a fixed interval $[t_0, T]$ with $N = (T-t_0)/h$ steps accumulates to $O(h^4)$, because the $N = O(1/h)$ steps each contribute $O(h^5)$ error. + +However, error accumulation depends on the stability of the ODE. For **stable** equations ($\text{Re}(\lambda) < 0$), errors are damped and the global error remains $O(h^4)$. For **unstable** equations ($\text{Re}(\lambda) > 0$), errors grow exponentially, and even small per-step errors can lead to wildly inaccurate solutions over long integration intervals. For such problems, reducing the step size and monitoring the solution for physical plausibility is essential. ## Practical Examples @@ -160,14 +248,25 @@ for (int i = 0; i < N.Length; i += 20) double halfLives = t / (Math.Log(2) / lambda); Console.WriteLine($"{t,4:F2} | {N[i],6:F1} | {halfLives,10:F3}"); } +// Print final time point (the loop step skips the last element) +{ + double t = (N.Length - 1) * dt; + double halfLives = t / (Math.Log(2) / lambda); + Console.WriteLine($"{t,4:F2} | {N[N.Length - 1],6:F1} | {halfLives,10:F3}"); +} ``` ### Example 2: Logistic Growth (Population Dynamics) -```cs -// Model: dP/dt = r*P*(1 - P/K) -// Where: r = growth rate, K = carrying capacity +The logistic equation models population growth with a carrying capacity: + +```math +\frac{dP}{dt} = rP\left(1 - \frac{P}{K}\right) +``` +The analytical solution is $P(t) = \frac{K}{1 + \left(\frac{K}{P_0} - 1\right)e^{-rt}}$, which has a sigmoidal shape. The growth rate $r$ controls how quickly the population approaches carrying capacity $K$. + +```cs double r = 0.5; // Growth rate double K = 1000.0; // Carrying capacity double P0 = 10.0; // Initial population @@ -197,27 +296,24 @@ for (int i = 0; i < P.Length; i++) } ``` -### Example 3: Harmonic Oscillator +### Example 3: Harmonic Oscillator (Systems of ODEs) -For second-order ODEs, convert to system of first-order ODEs: +> **Note:** The `RungeKutta` class handles scalar ODEs only (i.e., a single dependent variable). For systems of ODEs (multiple coupled equations), you must implement the RK4 stepping logic manually, as shown below. -```cs -// Second-order: d²y/dt² + ω²y = 0 -// Convert to system: -// dy₁/dt = y₂ -// dy₂/dt = -ω²y₁ -// Where y₁ = position, y₂ = velocity +A second-order ODE can always be converted to a system of first-order ODEs. For the harmonic oscillator $\frac{d^2y}{dt^2} + \omega^2 y = 0$, we introduce $y_1 = y$ (position) and $y_2 = \frac{dy}{dt}$ (velocity): + +```math +\frac{dy_1}{dt} = y_2, \qquad \frac{dy_2}{dt} = -\omega^2 y_1 +``` +The RK4 method is applied to both equations simultaneously, using the same $k$-values structure but evaluated for each component: + +```cs double omega = 2.0 * Math.PI; // Angular frequency (1 Hz) -// For systems, use multiple passes double y1_0 = 1.0; // Initial position double y2_0 = 0.0; // Initial velocity -Func dy1dt = (t, y1) => y2; // Needs y2 -Func dy2dt = (t, y2) => -omega * omega * y1; // Needs y1 - -// Need to track both variables manually int steps = 1000; double t0 = 0.0, t1 = 2.0; double dt = (t1 - t0) / (steps - 1); @@ -231,20 +327,20 @@ y2[0] = y2_0; for (int i = 1; i < steps; i++) { double t = t0 + (i - 1) * dt; - + // RK4 for system double k1_y1 = y2[i - 1]; double k1_y2 = -omega * omega * y1[i - 1]; - + double k2_y1 = y2[i - 1] + 0.5 * dt * k1_y2; double k2_y2 = -omega * omega * (y1[i - 1] + 0.5 * dt * k1_y1); - + double k3_y1 = y2[i - 1] + 0.5 * dt * k2_y2; double k3_y2 = -omega * omega * (y1[i - 1] + 0.5 * dt * k2_y1); - + double k4_y1 = y2[i - 1] + dt * k3_y2; double k4_y2 = -omega * omega * (y1[i - 1] + dt * k3_y1); - + y1[i] = y1[i - 1] + dt / 6.0 * (k1_y1 + 2 * k2_y1 + 2 * k3_y1 + k4_y1); y2[i] = y2[i - 1] + dt / 6.0 * (k1_y2 + 2 * k2_y2 + 2 * k3_y2 + k4_y2); } @@ -260,12 +356,18 @@ for (int i = 0; i < steps; i += 100) ### Example 4: Predator-Prey (Lotka-Volterra) -Classic ecology model: +The Lotka-Volterra equations model the dynamics of two interacting species: -```cs -// dx/dt = αx - βxy (prey) -// dy/dt = δxy - γy (predator) +```math +\frac{dx}{dt} = \alpha x - \beta xy \qquad \text{(prey)} +``` +```math +\frac{dy}{dt} = \delta xy - \gamma y \qquad \text{(predator)} +``` +The prey population grows exponentially in the absence of predators (rate $\alpha$) but is reduced by predation (rate $\beta xy$). Predators grow when prey is abundant (rate $\delta xy$) but die in its absence (rate $\gamma y$). The system exhibits periodic oscillations — predator peaks follow prey peaks with a phase lag. + +```cs double alpha = 1.0; // Prey growth rate double beta = 0.1; // Predation rate double delta = 0.075; // Predator growth from predation @@ -287,26 +389,26 @@ y[0] = y0; for (int i = 1; i < steps; i++) { double t = t0 + (i - 1) * dt; - + // RK4 for Lotka-Volterra system double k1_x = alpha * x[i - 1] - beta * x[i - 1] * y[i - 1]; double k1_y = delta * x[i - 1] * y[i - 1] - gamma * y[i - 1]; - - double k2_x = alpha * (x[i - 1] + 0.5 * dt * k1_x) - + + double k2_x = alpha * (x[i - 1] + 0.5 * dt * k1_x) - beta * (x[i - 1] + 0.5 * dt * k1_x) * (y[i - 1] + 0.5 * dt * k1_y); - double k2_y = delta * (x[i - 1] + 0.5 * dt * k1_x) * (y[i - 1] + 0.5 * dt * k1_y) - + double k2_y = delta * (x[i - 1] + 0.5 * dt * k1_x) * (y[i - 1] + 0.5 * dt * k1_y) - gamma * (y[i - 1] + 0.5 * dt * k1_y); - - double k3_x = alpha * (x[i - 1] + 0.5 * dt * k2_x) - + + double k3_x = alpha * (x[i - 1] + 0.5 * dt * k2_x) - beta * (x[i - 1] + 0.5 * dt * k2_x) * (y[i - 1] + 0.5 * dt * k2_y); - double k3_y = delta * (x[i - 1] + 0.5 * dt * k2_x) * (y[i - 1] + 0.5 * dt * k2_y) - + double k3_y = delta * (x[i - 1] + 0.5 * dt * k2_x) * (y[i - 1] + 0.5 * dt * k2_y) - gamma * (y[i - 1] + 0.5 * dt * k2_y); - - double k4_x = alpha * (x[i - 1] + dt * k3_x) - + + double k4_x = alpha * (x[i - 1] + dt * k3_x) - beta * (x[i - 1] + dt * k3_x) * (y[i - 1] + dt * k3_y); - double k4_y = delta * (x[i - 1] + dt * k3_x) * (y[i - 1] + dt * k3_y) - + double k4_y = delta * (x[i - 1] + dt * k3_x) * (y[i - 1] + dt * k3_y) - gamma * (y[i - 1] + dt * k3_y); - + x[i] = x[i - 1] + dt / 6.0 * (k1_x + 2 * k2_x + 2 * k3_x + k4_x); y[i] = y[i - 1] + dt / 6.0 * (k1_y + 2 * k2_y + 2 * k3_y + k4_y); } @@ -322,36 +424,47 @@ for (int i = 0; i < steps; i += 100) ## Choosing a Method -| Method | Order | When to Use | -|--------|-------|-------------| -| **Second-Order** | O(h²) | Simple problems, rough estimates | -| **Fourth-Order** | O(h⁴) | General purpose, good accuracy | -| **Fehlberg** | O(h⁵) | Adaptive control, stiff equations | -| **Cash-Karp** | O(h⁵) | Alternative adaptive method | +| Method | Order | Error per Step | When to Use | +|--------|-------|----------------|-------------| +| **Second-Order** | $O(h^2)$ | $O(h^3)$ | Simple problems, rough estimates | +| **Fourth-Order** | $O(h^4)$ | $O(h^5)$ | General purpose, good accuracy-to-cost ratio | +| **Fehlberg** | $O(h^5)$ | Embedded error | Adaptive control, variable dynamics | +| **Cash-Karp** | $O(h^5)$ | Embedded error | Alternative adaptive method | **Step Size Guidelines:** -- Fixed step: Choose dt such that solution is smooth -- Adaptive: Set tolerance based on required accuracy -- Stiff equations: Use smaller steps or adaptive methods -- For stability: dt < 2/|λ_max| where λ_max is largest eigenvalue +- Fixed step: Choose $h$ such that the solution is smooth and verify by halving $h$ +- Adaptive: Set tolerance based on required accuracy — the method handles the rest +- For stability: $h < 2.785 / |\lambda_{\max}|$ where $\lambda_{\max}$ is the largest eigenvalue magnitude of the Jacobian ## Best Practices -1. **Verify with known solutions** when possible -2. **Check convergence** by halving step size -3. **Use adaptive methods** for variable dynamics -4. **Monitor conservation** (energy, mass) if applicable -5. **Plot solutions** to detect instabilities -6. **Consider stability** for stiff equations +1. **Verify with known solutions** when possible — compare against analytical results for test problems +2. **Check convergence** by halving the step size and confirming the solution doesn't change significantly (the error should decrease by $2^4 = 16 \times$ for RK4) +3. **Use adaptive methods** for problems with variable dynamics — they automatically concentrate effort where the solution changes rapidly +4. **Monitor conservation** — for Hamiltonian systems (e.g., harmonic oscillator), check that energy is conserved over long integrations +5. **Plot solutions** to detect instabilities — oscillations growing in amplitude usually indicate the step size is too large +6. **Consider reformulation** for stiff equations — if possible, analytically eliminate the fast component or use a change of variables ## Limitations - Fixed-step methods require careful step size selection -- Stiff equations may require specialized solvers -- Systems require manual coupling of equations -- No built-in event detection -- For very high accuracy, consider specialized ODE libraries +- Very stiff equations may require specialized implicit solvers (not currently in the library) +- Systems of ODEs require manual coupling of equations — the `RungeKutta` class handles scalar ODEs only +- No built-in event detection (e.g., finding when the solution crosses zero) +- For very high accuracy or long-time integrations, consider symplectic integrators for Hamiltonian systems + +--- + +## References + +[1] E. Fehlberg, "Low-order classical Runge-Kutta formulas with stepsize control and their application to some heat transfer problems," NASA Technical Report 315, 1969. + +[2] J. R. Cash and A. H. Karp, "A variable order Runge-Kutta method for initial value problems with rapidly varying right-hand sides," *ACM Transactions on Mathematical Software*, vol. 16, no. 3, pp. 201-222, 1990. + +[3] W. H. Press, S. A. Teukolsky, W. T. Vetterling and B. P. Flannery, *Numerical Recipes: The Art of Scientific Computing*, 3rd ed., Cambridge, UK: Cambridge University Press, 2007. + +[4] J. C. Butcher, *Numerical Methods for Ordinary Differential Equations*, 3rd ed., Chichester: Wiley, 2016. --- -[← Previous: Special Functions](special-functions.md) | [Back to Index](../index.md) | [Next: Hypothesis Tests →](../statistics/hypothesis-tests.md) +[← Previous: Special Functions](special-functions.md) | [Back to Index](../index.md) | [Next: Interpolation →](../data/interpolation.md) diff --git a/docs/mathematics/optimization.md b/docs/mathematics/optimization.md index 8849c688..433d24fb 100644 --- a/docs/mathematics/optimization.md +++ b/docs/mathematics/optimization.md @@ -1,6 +1,6 @@ # Optimization -[← Previous: Numerical Differentiation](differentiation.md) | [Back to Index](../index.md) | [Next: Linear Algebra →](linear-algebra.md) +[← Previous: Numerical Differentiation](differentiation.md) | [Back to Index](../index.md) | [Next: Root Finding →](root-finding.md) Optimization is the process of finding the parameter set that minimizes (or maximizes) an objective function. The ***Numerics*** library provides a comprehensive suite of local and global optimization algorithms for both unconstrained and constrained problems. These methods are essential for parameter estimation, model calibration, machine learning, and engineering design optimization. @@ -150,7 +150,7 @@ Simple gradient-based optimization with line search: ```cs var gd = new GradientDescent(Rosenbrock, 2, initial, lower, upper); -gd.LearningRate = 0.001; // Step size +gd.Alpha = 0.001; // Step size (learning rate) gd.Minimize(); ``` @@ -160,7 +160,7 @@ Adaptive Moment Estimation, popular in machine learning applications [[3]](#3): ```cs var adam = new ADAM(Rosenbrock, 2, initial, lower, upper); -adam.LearningRate = 0.001; +adam.Alpha = 0.001; adam.Beta1 = 0.9; // First moment decay adam.Beta2 = 0.999; // Second moment decay adam.Minimize(); @@ -181,9 +181,7 @@ Uses the golden ratio to bracket the minimum: ```cs Func f1d = x => Math.Pow(x - 2, 2) + 3; -var golden = new GoldenSection(f1d, 1); -golden.LowerBounds = new[] { 0.0 }; -golden.UpperBounds = new[] { 5.0 }; +var golden = new GoldenSection(f1d, 0.0, 5.0); golden.Minimize(); Console.WriteLine($"Minimum at x = {golden.BestParameterSet.Values[0]:F6}"); @@ -194,9 +192,7 @@ Console.WriteLine($"Minimum at x = {golden.BestParameterSet.Values[0]:F6}"); Combines golden section search with parabolic interpolation for faster convergence: ```cs -var brent = new BrentSearch(f1d, 1); -brent.LowerBounds = new[] { 0.0 }; -brent.UpperBounds = new[] { 5.0 }; +var brent = new BrentSearch(f1d, 0.0, 5.0); brent.Minimize(); ``` @@ -206,7 +202,31 @@ Global optimization methods are designed to find the global minimum across the e ### Differential Evolution -Differential Evolution (DE) is a population-based evolutionary algorithm that's very robust for continuous optimization [[4]](#4). It creates trial vectors by combining existing population members. +Differential Evolution (DE) is a population-based evolutionary algorithm that's very robust for continuous optimization [[4]](#4). For each member $\mathbf{x}_i$ of the population, DE creates a **trial vector** $\mathbf{u}$ using mutation and crossover: + +**Mutation** (DE/rand/1): Three distinct population members $\mathbf{x}_{r_0}$, $\mathbf{x}_{r_1}$, $\mathbf{x}_{r_2}$ are randomly selected, and a **mutant vector** is formed: + +```math +\mathbf{v} = \mathbf{x}_{r_0} + G \cdot (\mathbf{x}_{r_1} - \mathbf{x}_{r_2}) +``` + +where $G$ is a scale factor. In the ***Numerics*** implementation, $G$ is **dithered** with 90% probability: $G = 0.5 + \text{rand}() \times 0.5$, which helps avoid stagnation. + +**Crossover** (binomial): The trial vector $\mathbf{u}$ is assembled component-by-component: + +```math +u_j = \begin{cases} v_j & \text{if } \text{rand}() \leq CR \text{ or } j = j_{\text{rand}} \\ x_{i,j} & \text{otherwise} \end{cases} +``` + +where $CR$ is the crossover probability and $j_{\text{rand}}$ is a randomly chosen index that ensures at least one component comes from the mutant vector. + +**Selection** (greedy): The trial vector replaces the target only if it has equal or better fitness: + +```math +\mathbf{x}_i^{(g+1)} = \begin{cases} \mathbf{u} & \text{if } f(\mathbf{u}) \leq f(\mathbf{x}_i^{(g)}) \\ \mathbf{x}_i^{(g)} & \text{otherwise} \end{cases} +``` + +The algorithm converges when the standard deviation of fitness values across the population falls below the tolerance. ```cs using Numerics.Mathematics.Optimization; @@ -228,7 +248,7 @@ var upper = new double[] { 5.12, 5.12 }; var de = new DifferentialEvolution(Rastrigin, 2, lower, upper); de.PopulationSize = 50; de.CrossoverProbability = 0.9; -de.DifferentialWeight = 0.8; +de.Mutation = 0.75; de.MaxIterations = 1000; de.Minimize(); @@ -241,39 +261,68 @@ Console.WriteLine($"Function value: {de.BestParameterSet.Fitness:F10}"); **Parameters**: - `PopulationSize`: Number of candidate solutions (default = 10 × dimensions) - `CrossoverProbability`: Probability of crossover (default = 0.9) -- `DifferentialWeight`: Scaling factor for mutation (default = 0.8) +- `Mutation`: Scaling factor for mutation (default = 0.75) ### Particle Swarm Optimization -PSO simulates social behavior of bird flocking or fish schooling [[5]](#5). Particles move through the search space influenced by their own best position and the swarm's best position. +PSO simulates social behavior of bird flocking or fish schooling [[5]](#5). Each particle $i$ has a position $\mathbf{x}_i$ and velocity $\mathbf{v}_i$ that are updated at each iteration using: + +```math +v_{i,j}^{(t+1)} = w \cdot v_{i,j}^{(t)} + c_1 \cdot r_1 \cdot (p_{i,j} - x_{i,j}^{(t)}) + c_2 \cdot r_2 \cdot (g_j - x_{i,j}^{(t)}) +``` +```math +x_{i,j}^{(t+1)} = x_{i,j}^{(t)} + v_{i,j}^{(t+1)} +``` + +where: +- $w$ is the **inertia weight**, which decreases linearly from $w_{\max} = 0.9$ to $w_{\min} = 0.4$ over the optimization, balancing exploration (high $w$) and exploitation (low $w$) +- $c_1 = c_2 = 2.05$ are the cognitive and social acceleration coefficients +- $r_1, r_2 \sim U(0,1)$ are independent random numbers (per component, per particle) +- $\mathbf{p}_i$ is particle $i$'s personal best position (best position it has visited) +- $\mathbf{g}$ is the global best position (best position any particle has visited) + +The three terms represent **momentum** (continue in the current direction), **cognitive pull** (return toward personal best), and **social pull** (move toward the swarm's best). ```cs var pso = new ParticleSwarm(Rastrigin, 2, lower, upper); pso.PopulationSize = 40; -pso.InertiaWeight = 0.7; -pso.CognitiveWeight = 1.5; // Personal best influence -pso.SocialWeight = 1.5; // Global best influence pso.Minimize(); Console.WriteLine($"Solution: [{pso.BestParameterSet.Values[0]:F6}, {pso.BestParameterSet.Values[1]:F6}]"); ``` -**Advantages**: Fast, simple to implement, works well for continuous problems. +**Advantages**: Fast convergence, simple concept, works well for continuous problems. **Parameters**: -- `PopulationSize`: Number of particles (default = 10 × dimensions) -- `InertiaWeight`: Momentum (default = 0.7) -- `CognitiveWeight`: Personal best attraction (default = 1.5) -- `SocialWeight`: Global best attraction (default = 1.5) +- `PopulationSize`: Number of particles (default = 30) ### Shuffled Complex Evolution (SCE-UA) -SCE-UA was specifically developed for calibrating hydrological models [[6]](#6). It combines complex shuffling with competitive evolution. +SCE-UA was specifically developed for calibrating hydrological models [[6]](#6). The algorithm partitions the population into $p$ **complexes**, evolves each complex independently using a **Competitive Complex Evolution** (CCE) strategy, then shuffles members between complexes to share information. + +The CCE step within each complex proceeds as follows: + +1. **Sub-complex selection**: Select $q = D+1$ points from the complex using a trapezoidal probability distribution that favors better-ranked points: $P(i) = \frac{2(N+1-i)}{N(N+1)}$ + +2. **Reflection**: Compute the centroid $\mathbf{g}$ of all sub-complex points except the worst point $\mathbf{x}_w$, then reflect: + +```math +\mathbf{r} = 2\mathbf{g} - \mathbf{x}_w +``` + +3. **Contraction**: If the reflected point is infeasible or worse than $\mathbf{x}_w$, try contraction: + +```math +\mathbf{c} = \frac{\mathbf{g} + \mathbf{x}_w}{2} +``` + +4. **Mutation**: If contraction also fails, generate a random point within the smallest bounding box of the complex. + +The shuffling step redistributes points across complexes, preventing any single complex from stagnating. This combination of local competitive evolution within complexes and global information sharing between complexes makes SCE-UA particularly effective for the rugged, multimodal objective functions typical of hydrological model calibration. ```cs var sce = new ShuffledComplexEvolution(Rastrigin, 2, lower, upper); -sce.NumberOfComplexes = 5; -sce.ComplexSize = 10; +sce.Complexes = 5; sce.MaxIterations = 1000; sce.Minimize(); @@ -290,7 +339,7 @@ SA mimics the physical process of annealing in metallurgy [[7]](#7). It accepts ```cs var sa = new SimulatedAnnealing(Rastrigin, 2, lower, upper); -sa.InitialTemperature = 100.0; +sa.InitialTemperature = 10.0; sa.CoolingRate = 0.95; sa.MaxIterations = 10000; sa.Minimize(); @@ -301,7 +350,7 @@ Console.WriteLine($"Solution: [{sa.BestParameterSet.Values[0]:F6}, {sa.BestParam **Advantages**: Can escape local minima, works for discrete and continuous problems. **Parameters**: -- `InitialTemperature`: Starting temperature (default = 100) +- `InitialTemperature`: Starting temperature (default = 10) - `CoolingRate`: Temperature reduction factor (default = 0.95) ### Multi-Start Optimization @@ -309,9 +358,9 @@ Console.WriteLine($"Solution: [{sa.BestParameterSet.Values[0]:F6}, {sa.BestParam Combines local search with multiple random starting points: ```cs -var ms = new MultiStart(Rastrigin, 2, lower, upper); -ms.LocalMethod = LocalMethod.BFGS; // Choose local optimizer -ms.NumberOfStarts = 20; +var initial = new double[] { 0.0, 0.0 }; +var ms = new MultiStart(Rastrigin, 2, initial, lower, upper, LocalMethod.BFGS); +ms.MaxIterations = 20; // Number of random starts ms.Minimize(); Console.WriteLine($"Best solution: [{ms.BestParameterSet.Values[0]:F6}, {ms.BestParameterSet.Values[1]:F6}]"); @@ -326,8 +375,8 @@ Console.WriteLine($"Best solution: [{ms.BestParameterSet.Values[0]:F6}, {ms.Best Clustering-based global optimization that avoids redundant local searches: ```cs -var mlsl = new MLSL(Rastrigin, 2, lower, upper); -mlsl.LocalMethod = LocalMethod.BFGS; +var initial = new double[] { 0.0, 0.0 }; +var mlsl = new MLSL(Rastrigin, 2, initial, lower, upper, LocalMethod.BFGS); mlsl.Minimize(); ``` @@ -350,7 +399,7 @@ double Objective(double[] x) // Constraint: x + y >= 1 var constraint = new Constraint( - x => x[0] + x[1] - 1, // g(x) >= 0 form + x => x[0] + x[1], 2, 1.0, // g(x) >= value form ConstraintType.GreaterThanOrEqualTo ); @@ -358,9 +407,11 @@ var lower = new double[] { -5, -5 }; var upper = new double[] { 5, 5 }; var initial = new double[] { 0, 0 }; -var al = new AugmentedLagrange(Objective, 2, initial, lower, upper); -al.AddConstraint(constraint); -al.LocalMethod = LocalMethod.BFGS; // Local optimizer for subproblems +// Create the inner optimizer +var bfgs = new BFGS(Objective, 2, initial, lower, upper); + +// Create the Augmented Lagrange optimizer with constraints +var al = new AugmentedLagrange(Objective, bfgs, new IConstraint[] { constraint }); al.MaxIterations = 100; al.Minimize(); @@ -370,7 +421,7 @@ Console.WriteLine($"Constraint satisfied: {al.BestParameterSet.Values[0] + al.Be **Constraint Types**: - `ConstraintType.EqualTo`: Equality constraint $g(\mathbf{x}) = 0$ -- `ConstraintType.LessThanOrEqualTo`: Inequality constraint $g(\mathbf{x}) \leq 0$ +- `ConstraintType.LesserThanOrEqualTo`: Inequality constraint $g(\mathbf{x}) \leq 0$ - `ConstraintType.GreaterThanOrEqualTo`: Inequality constraint $g(\mathbf{x}) \geq 0$ **Example: Minimize subject to multiple constraints**: @@ -386,15 +437,14 @@ double ObjectiveFunc(double[] x) return Math.Pow(x[0] - 3, 2) + Math.Pow(x[1] - 2, 2); } -var c1 = new Constraint(x => 5 - x[0] - x[1], ConstraintType.GreaterThanOrEqualTo); -var c2 = new Constraint(x => x[0] - 1, ConstraintType.GreaterThanOrEqualTo); -var c3 = new Constraint(x => x[1] - 1, ConstraintType.GreaterThanOrEqualTo); +var c1 = new Constraint(x => x[0] + x[1], 2, 5.0, ConstraintType.LesserThanOrEqualTo); +var c2 = new Constraint(x => x[0], 2, 1.0, ConstraintType.GreaterThanOrEqualTo); +var c3 = new Constraint(x => x[1], 2, 1.0, ConstraintType.GreaterThanOrEqualTo); -var constrained = new AugmentedLagrange(ObjectiveFunc, 2, new[] { 2.0, 2.0 }, - new[] { 0.0, 0.0 }, new[] { 10.0, 10.0 }); -constrained.AddConstraint(c1); -constrained.AddConstraint(c2); -constrained.AddConstraint(c3); +var innerOptimizer = new BFGS(ObjectiveFunc, 2, new[] { 2.0, 2.0 }, + new[] { 0.0, 0.0 }, new[] { 10.0, 10.0 }); +var constrained = new AugmentedLagrange(ObjectiveFunc, innerOptimizer, + new IConstraint[] { c1, c2, c3 }); constrained.Minimize(); Console.WriteLine($"Constrained optimum: [{constrained.BestParameterSet.Values[0]:F4}, " + @@ -429,7 +479,7 @@ double ObjectiveFunction(double[] parameters) } // Compute RMSE - double rmse = Statistics.RMSE(observed, simulated); + double rmse = GoodnessOfFit.RMSE(observed, simulated); return rmse; } @@ -439,7 +489,7 @@ var upper = new double[] { 5.0, 3.0 }; // C <= 5.0, α <= 3.0 // Use SCE-UA (recommended for hydrological calibration) var optimizer = new ShuffledComplexEvolution(ObjectiveFunction, 2, lower, upper); -optimizer.NumberOfComplexes = 5; +optimizer.Complexes = 5; optimizer.MaxIterations = 1000; optimizer.Minimize(); @@ -457,7 +507,7 @@ for (int i = 0; i < observed.Length; i++) Math.Pow(precipitation[i], optimizer.BestParameterSet.Values[1]); } -double nse = Statistics.NSE(observed, final_simulated); +double nse = GoodnessOfFit.NashSutcliffeEfficiency(observed, final_simulated); Console.WriteLine($" NSE = {nse:F4}"); ``` @@ -525,7 +575,14 @@ The `BestParameterSet` contains: ### Hessian Matrix -When `ComputeHessian = true`, the Hessian at the solution is computed numerically. This provides information about parameter sensitivity and uncertainty: +When `ComputeHessian = true`, the Hessian at the solution is computed numerically. The Hessian $\mathbf{H}$ is the matrix of second partial derivatives of the objective function, and its eigenvalues reveal important information about the solution: + +- **All eigenvalues positive**: The solution is a local minimum (the Hessian is positive definite) +- **Large eigenvalue**: The objective is highly curved in that direction — the corresponding parameter is well-determined +- **Small eigenvalue**: The objective is nearly flat — the parameter is poorly determined or identifiable +- **Near-zero eigenvalue**: Indicates a ridge or valley in the objective surface, suggesting parameter correlation or redundancy + +The inverse of the Hessian approximates the covariance matrix of the parameters, so parameter standard errors can be estimated as $SE(\hat{\theta}_i) \approx \sqrt{|H_{ii}^{-1}|}$: ```cs optimizer.ComputeHessian = true; @@ -551,7 +608,7 @@ if (optimizer.Status == OptimizationStatus.Success) { Console.WriteLine("Optimization converged successfully"); } -else if (optimizer.Status == OptimizationStatus.MaxIterationsReached) +else if (optimizer.Status == OptimizationStatus.MaximumIterationsReached) { Console.WriteLine("Maximum iterations reached - may not have converged"); } @@ -589,4 +646,4 @@ else if (optimizer.Status == OptimizationStatus.MaxIterationsReached) --- -[← Previous: Numerical Differentiation](differentiation.md) | [Back to Index](../index.md) | [Next: Linear Algebra →](linear-algebra.md) +[← Previous: Numerical Differentiation](differentiation.md) | [Back to Index](../index.md) | [Next: Root Finding →](root-finding.md) diff --git a/docs/mathematics/root-finding.md b/docs/mathematics/root-finding.md index 59b18c00..61d78f6d 100644 --- a/docs/mathematics/root-finding.md +++ b/docs/mathematics/root-finding.md @@ -39,10 +39,26 @@ The bisection method is the simplest root-finding algorithm. It repeatedly bisec 4. Otherwise, replace either $a$ or $b$ with $c$ based on the sign of $f(c)$ 5. Repeat until convergence +### Convergence Rate + +Bisection has **linear convergence** with a constant factor of $1/2$. After $n$ iterations, the error is bounded by: + +```math +|e_n| \leq \frac{b - a}{2^n} +``` + +where $b - a$ is the initial interval width. This means each iteration gains approximately one binary digit of accuracy. To achieve a tolerance $\varepsilon$, the number of iterations required is: + +```math +n \geq \frac{\log(b - a) - \log(\varepsilon)}{\log 2} +``` + +For example, starting with $[0, 3]$ and tolerance $10^{-10}$, bisection needs at most $\lceil \log_2(3 \times 10^{10}) \rceil = 35$ iterations. This predictability is one of bisection's strengths — you know exactly how many iterations you need before you start. + ### Usage ```cs -using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; // Find root of f(x) = x² - 4 (roots at x = ±2) Func f = x => x * x - 4; @@ -77,19 +93,108 @@ Console.WriteLine($"Verification: f({root}) = {f(root):E3}"); ## Brent's Method -Brent's method combines the robustness of bisection with the speed of secant and inverse quadratic interpolation [[2]](#2). It's generally the best general-purpose root finder. +Brent's method is the recommended general-purpose root-finding algorithm in the ***Numerics*** library. It combines the guaranteed convergence of bisection with the speed of the secant method and inverse quadratic interpolation, automatically switching between these strategies to achieve robust, fast convergence without requiring derivatives [[2]](#2) [[3]](#3). For the vast majority of root-finding problems, Brent's method should be the first method you reach for. -### Algorithm +### Mathematical Description + +Brent's method maintains a bracketing interval $[a, b]$ such that $f(a)$ and $f(b)$ have opposite signs, guaranteeing that a root exists within the interval by the Intermediate Value Theorem. At each iteration, it selects one of three strategies to propose the next approximation: + +**Bisection.** The simplest strategy: take the midpoint of the current bracket. + +```math +x_{\text{bisect}} = \frac{a + b}{2} +``` + +Bisection reduces the bracket by exactly half at each step, giving linear convergence with error bound $|e_n| \leq (b - a) / 2^n$. It is slow but absolutely reliable. + +**Secant method.** When only two distinct points are available (i.e., the previous contrapoint $a$ equals the older contrapoint $c$), the algorithm uses the secant formula, which fits a line through the two most recent function values and finds where it crosses zero: + +```math +x_{\text{secant}} = b - f(b) \cdot \frac{b - a}{f(b) - f(a)} +``` + +The secant method converges superlinearly with order approximately $\varphi \approx 1.618$ (the golden ratio) for well-behaved functions. + +**Inverse quadratic interpolation (IQI).** When three distinct points $a$, $b$, $c$ with distinct function values are available, the algorithm fits an inverse quadratic (a parabola through the three points with $x$ as a function of $y$) and evaluates it at $y = 0$: + +```math +x_{\text{IQI}} = \frac{f_b f_c \cdot a}{(f_a - f_b)(f_a - f_c)} + \frac{f_a f_c \cdot b}{(f_b - f_a)(f_b - f_c)} + \frac{f_a f_b \cdot c}{(f_c - f_a)(f_c - f_b)} +``` + +where $f_a = f(a)$, $f_b = f(b)$, $f_c = f(c)$. IQI can converge even faster than the secant method when the function is smooth and the iterates are close to the root. + +### How It Works: The Decision Logic + +The power of Brent's method lies in how it decides which strategy to use at each step. The algorithm follows a specific decision procedure that ensures it never loses the safety of bisection while taking faster steps whenever possible. Here is the logic as implemented in the ***Numerics*** library: + +1. **Maintain the bracket.** The algorithm tracks three points: $b$ (the current best estimate, where $|f(b)|$ is smallest), $a$ (the previous iterate), and $c$ (the contrapoint such that $f(b)$ and $f(c)$ have opposite signs). If $f(b)$ and $f(c)$ have the same sign, the contrapoint is reset to $a$. + +2. **Compute the midpoint and tolerance.** At each step, compute: + +```math +x_m = \frac{c - b}{2}, \qquad \text{tol}_1 = 2 \varepsilon_{\text{mach}} |b| + \frac{\text{tol}}{2} +``` -The method maintains a bracketing interval and uses: -- **Inverse quadratic interpolation** when three points are available -- **Secant method** when two points are available -- **Bisection** as a fallback to guarantee convergence +where $\varepsilon_{\text{mach}}$ is machine epsilon and $\text{tol}$ is the user-specified tolerance. If $|x_m| \leq \text{tol}_1$ or $f(b) = 0$, the root has been found. + +3. **Try a fast step.** If the previous step $e$ was large enough ($|e| \geq \text{tol}_1$) and $b$ is improving ($|f(a)| > |f(b)|$), then attempt an open method: + - If $a = c$ (only two distinct points available), use the **secant method**. + - If $a \neq c$ (three distinct points available), use **inverse quadratic interpolation**. + +4. **Accept or reject the fast step.** The proposed step $d = p/q$ is accepted only if it satisfies two conditions: + - The step must be smaller than three-quarters of the distance to the midpoint: $2|p| < 3 |x_m \cdot q| - |\text{tol}_1 \cdot q|$ + - The step must be smaller than half the previous step: $2|p| < |e \cdot q|$ + + These conditions ensure the method is making adequate progress toward the root. If either condition fails, the algorithm falls back to bisection. + +5. **Update.** Apply the accepted step (or bisection fallback) to produce the new iterate $b$, evaluate $f(b)$, and repeat. + +This design means that Brent's method takes fast steps when they are safe and productive, but always has bisection as a backstop, so it never diverges. + +### Convergence Properties + +Brent's method offers a rare combination of guaranteed convergence and fast practical performance: + +- **Guaranteed convergence.** Like bisection, the method always maintains a valid bracket around the root. It will converge to a root for any continuous function where the initial interval satisfies $f(a) \cdot f(b) < 0$, regardless of how ill-conditioned the function is. + +- **Superlinear convergence in practice.** For smooth, well-behaved functions, the algorithm typically achieves superlinear convergence by spending most iterations in secant or IQI mode. In the best case, IQI converges with order approximately 1.839. + +- **Worst case is bisection.** When the function is poorly behaved (discontinuous derivatives, sharp curvature changes, or near-flat regions), the method gracefully degrades to bisection's linear convergence rate of $O(1/2^n)$. This is a floor, not a failure. + +- **No pathological failure modes.** Unlike Newton-Raphson, which can cycle, diverge, or overshoot for bad initial guesses, and unlike the secant method, which can leave the bracket entirely, Brent's method has no failure modes beyond exceeding the maximum iteration count. + +The convergence tolerance in the ***Numerics*** implementation uses a combined criterion: the root is accepted when the bracket half-width $|x_m|$ is within $\text{tol}_1 = 2\varepsilon_{\text{mach}}|b| + \text{tol}/2$, or when $f(b) = 0$ exactly. This accounts for both absolute and relative precision near the root. + +### Why Brent's Method Is the Recommended Default + +For solving $f(x) = 0$ in a single variable, Brent's method should be the default choice because: + +- **No derivatives needed.** Unlike Newton-Raphson, Brent's method requires only function evaluations. There is no need to derive, implement, or numerically approximate $f'(x)$, which is a significant practical advantage. + +- **Guaranteed to converge.** Given a valid bracketing interval where $f(a)$ and $f(b)$ have opposite signs, the method will always find a root. Newton-Raphson and the secant method have no such guarantee. + +- **Typically faster than bisection.** While bisection requires approximately $\log_2((b-a)/\varepsilon)$ iterations, Brent's method usually converges in far fewer iterations by exploiting the smoothness of the function. For the test function $f(x) = x^3 - 2x - 5$ on $[2, 3]$, bisection needs 27 iterations while Brent needs only 5. + +- **No pathological failure cases.** Newton-Raphson can diverge, oscillate, or cycle when the initial guess is poor, the derivative is near zero, or the function has inflection points. Brent's method avoids all of these failure modes. + +- **Handles both well-behaved and ill-conditioned functions.** The adaptive switching between IQI, secant, and bisection means the algorithm performs well across a wide range of function behaviors without requiring the user to diagnose the function's properties in advance. + +### When NOT to Use Brent's Method + +While Brent's method is the best general-purpose choice, there are situations where a different method is more appropriate: + +- **Derivatives are available and the function is smooth.** If you already have an analytical derivative $f'(x)$ and the function is smooth with a good initial guess, Newton-Raphson will converge faster (quadratically vs. superlinearly). For functions where the derivative is cheap to evaluate, the `NewtonRaphson.RobustSolve` method provides Newton's speed with bisection's safety net. + +- **Systems of equations or multiple dimensions.** Brent's method is strictly a univariate solver. For systems of nonlinear equations $\mathbf{F}(\mathbf{x}) = \mathbf{0}$, use multidimensional optimization methods such as Nelder-Mead or Newton-based nonlinear solvers. + +- **The root is not bracketed.** Brent's method requires an initial interval $[a, b]$ where $f(a)$ and $f(b)$ have opposite signs. If you cannot identify such a bracket, consider using the `Brent.Bracket` helper method (see below), the secant method, or Newton-Raphson which do not require bracketing. + +- **The function does not change sign at the root.** For roots of even multiplicity (e.g., $f(x) = x^2$ has a root at $x = 0$ but does not change sign), bracketing methods cannot be used. In these cases, reformulate the problem or use Newton-Raphson on $g(x) = f(x)/f'(x)$. ### Usage ```cs -using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; // Find root of f(x) = cos(x) - x (root around x ≈ 0.739) Func f = x => Math.Cos(x) - x; @@ -104,7 +209,44 @@ Console.WriteLine($"Root: {root:F12}"); // 0.739085133215 Console.WriteLine($"Verification: f({root}) = {f(root):E3}"); ``` -### Example: Finding where two functions intersect +The `Solve` method accepts the following parameters: + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `f` | `Func` | (required) | The function to solve | +| `lowerBound` | `double` | (required) | Lower bound $a$ of the bracketing interval | +| `upperBound` | `double` | (required) | Upper bound $b$ of the bracketing interval | +| `tolerance` | `double` | `1e-8` | Desired tolerance for the root | +| `maxIterations` | `int` | `1000` | Maximum number of iterations | +| `reportFailure` | `bool` | `true` | If true, throws an exception on failure; if false, returns the last iterate | + +### Automatic Bracket Expansion + +When you are unsure whether your initial interval brackets the root, the `Brent.Bracket` helper method can expand the interval outward until a sign change is found. It repeatedly expands the interval by a factor of 1.6, testing the function value at each new endpoint: + +```cs +using Numerics.Mathematics.RootFinding; + +// We think the root is near x = 5, but are not sure of the bracket +Func f = x => x * x - 25; + +double a = 4.0, b = 6.0; +double fa, fb; + +bool found = Brent.Bracket(f, ref a, ref b, out fa, out fb, maxIterations: 10); + +if (found) +{ + double root = Brent.Solve(f, a, b); + Console.WriteLine($"Root: {root:F10}"); // 5.0000000000 +} +else +{ + Console.WriteLine("Could not bracket the root."); +} +``` + +### Example: Finding Where Two Functions Intersect To find where $f(x) = g(x)$, solve $h(x) = f(x) - g(x) = 0$: @@ -121,19 +263,74 @@ Console.WriteLine($"At this point: f({intersection}) = {f(intersection):F6}"); Console.WriteLine($"At this point: g({intersection}) = {g(intersection):F6}"); ``` -### Advantages and Disadvantages +### Example: Hydrologic Design with Manning's Equation -**Advantages:** -- Very fast convergence (superlinear) -- Guaranteed to converge with bracketing interval -- Automatically switches between methods for optimal performance -- Widely considered the best general-purpose method +A common civil engineering problem: determine the water depth in a trapezoidal channel that produces a given discharge. Manning's equation relates flow rate to channel geometry: -**Disadvantages:** -- Requires bracketing interval -- More complex than simpler methods +```math +Q = \frac{1}{n} A R^{2/3} S^{1/2} +``` + +where $Q$ is discharge, $n$ is Manning's roughness coefficient, $A$ is cross-sectional area, $R = A/P$ is hydraulic radius, $P$ is wetted perimeter, and $S$ is channel slope. For a trapezoidal channel with bottom width $b_w$ and side slope $z$: + +```math +A = y(b_w + z \cdot y), \qquad P = b_w + 2y\sqrt{1 + z^2} +``` -**When to use:** Default choice for most root-finding problems. +Given a target discharge, solve for the normal depth $y$: + +```cs +using Numerics.Mathematics.RootFinding; + +// Channel parameters +double n = 0.030; // Manning's roughness (natural channel) +double bw = 10.0; // Bottom width (m) +double z = 2.0; // Side slope (horizontal:vertical) +double S = 0.001; // Channel slope +double Qtarget = 50; // Target discharge (m³/s) + +// Manning's equation residual: f(y) = Q(y) - Qtarget +Func f = y => +{ + double A = y * (bw + z * y); + double P = bw + 2 * y * Math.Sqrt(1 + z * z); + double R = A / P; + double Q = (1.0 / n) * A * Math.Pow(R, 2.0 / 3.0) * Math.Sqrt(S); + return Q - Qtarget; +}; + +// Solve for normal depth +double depth = Brent.Solve(f, + lowerBound: 0.01, // Minimum physical depth + upperBound: 20.0, // Maximum reasonable depth + tolerance: 1e-6); + +Console.WriteLine($"Normal depth: {depth:F4} m"); +Console.WriteLine($"Verification: Q = {f(depth) + Qtarget:F4} m³/s"); +``` + +### Example: Controlling Failure Behavior + +In production code, you may want to handle convergence failure gracefully rather than allowing exceptions: + +```cs +using Numerics.Mathematics.RootFinding; + +Func f = x => Math.Cos(x) - x; + +// Suppress exception on failure — returns last iterate instead +double root = Brent.Solve(f, 0, 1, + tolerance: 1e-15, // Very tight tolerance + maxIterations: 5, // Very few iterations allowed + reportFailure: false); // Don't throw on failure + +// Always verify the result +double residual = Math.Abs(f(root)); +if (residual < 1e-10) + Console.WriteLine($"Root found: {root:F12}"); +else + Console.WriteLine($"Warning: residual {residual:E3} exceeds acceptable threshold"); +``` ## Secant Method @@ -153,10 +350,20 @@ This approximates the derivative as: f'(x_n) \approx \frac{f(x_n) - f(x_{n-1})}{x_n - x_{n-1}} ``` +### Convergence Rate + +The secant method has **superlinear convergence** of order $\varphi = (1 + \sqrt{5})/2 \approx 1.618$ (the golden ratio). Near a simple root $x^*$, the error satisfies: + +```math +|e_{n+1}| \approx \left|\frac{f''(x^*)}{2f'(x^*)}\right|^{\varphi - 1} |e_n|^{\varphi} +``` + +This is faster than bisection's linear convergence but slower than Newton's quadratic convergence. However, since the secant method requires only **one function evaluation per iteration** (compared to Newton's two — one for $f$ and one for $f'$), it can be more efficient overall. In terms of function evaluations, the secant method's efficiency index is $\varphi^{1/1} \approx 1.618$, while Newton's is $2^{1/2} \approx 1.414$. + ### Usage ```cs -using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; // Find root of f(x) = x³ - 2x - 5 Func f = x => x * x * x - 2 * x - 5; @@ -199,10 +406,38 @@ x_{n+1} = x_n - \frac{f(x_n)}{f'(x_n)} Geometrically, this finds where the tangent line at $(x_n, f(x_n))$ crosses the x-axis. +### Convergence Rate + +Newton-Raphson has **quadratic convergence** near a simple root. If $e_n = x_n - x^*$ is the error at step $n$, then: + +```math +e_{n+1} = -\frac{f''(x^*)}{2f'(x^*)} \cdot e_n^2 + O(e_n^3) +``` + +This means the number of correct digits roughly **doubles** each iteration. Starting from $|e_0| = 0.1$, after 4 iterations the error is approximately $10^{-16}$ — near machine precision. This is why Newton's method typically converges in 4–6 iterations. + +The constant $C = |f''(x^*)|/(2|f'(x^*)|)$ determines how fast convergence kicks in. When $|f''|$ is large relative to $|f'|$ at the root (i.e., the function curves sharply while crossing zero slowly), the method needs a closer starting point for quadratic convergence to dominate. + +For **repeated roots** where $f(x^*) = f'(x^*) = 0$, convergence degrades to linear. In such cases, modified Newton's method using $g(x) = f(x)/f'(x)$ restores quadratic convergence. + +### Basin of Attraction + +Newton's method is not globally convergent — its behavior depends critically on the initial guess. The **basin of attraction** for a root $x^*$ is the set of starting points $x_0$ from which Newton's method converges to $x^*$. + +For polynomials with multiple roots, basins of attraction can have fractal boundaries. Even simple functions can produce surprising behavior: + +- For $f(x) = x^3 - 1$ (with roots at $1$, $e^{2\pi i/3}$, $e^{4\pi i/3}$ in the complex plane), the boundary between basins of attraction is a fractal — the Newton fractal. + +- For $f(x) = x^3 - 2x + 2$, starting at $x_0 = 0$ produces the cycle $0 \to 1 \to 1 \to \ldots$ that never converges. + +- For $f(x) = |x|^a$ where $0 < a < 1/2$, Newton's method diverges from every starting point except the root itself. + +A practical rule of thumb: Newton's method converges when $|f(x_0) \cdot f''(x_0)| < |f'(x_0)|^2$ (the Newton-Kantorovich condition). When unsure about the initial guess, use the `RobustSolve` method which falls back to bisection. + ### Usage ```cs -using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; // Find root of f(x) = x² - 2 (square root of 2) Func f = x => x * x - 2; @@ -221,6 +456,7 @@ Console.WriteLine($"Verification: {root}² = {root * root:F12}"); ```cs using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; Func f = x => x * x - 2; Func df = x => NumericalDerivative.Derivative(f, x); @@ -263,16 +499,18 @@ This method uses Newton-Raphson when it's making good progress, but falls back t A practical example from financial mathematics - solving the Black-Scholes equation for implied volatility: ```cs -using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; +using Numerics.Mathematics.SpecialFunctions; // Black-Scholes call option price double BlackScholesCall(double S, double K, double T, double r, double sigma) { double d1 = (Math.Log(S / K) + (r + 0.5 * sigma * sigma) * T) / (sigma * Math.Sqrt(T)); double d2 = d1 - sigma * Math.Sqrt(T); - - double N(double x) => 0.5 * (1.0 + Erf(x / Math.Sqrt(2))); - + + // Standard normal CDF using the error function + double N(double x) => 0.5 * (1.0 + Erf.Function(x / Math.Sqrt(2))); + return S * N(d1) - K * Math.Exp(-r * T) * N(d2); } @@ -298,7 +536,7 @@ Console.WriteLine($"Verification: BS price = {BlackScholesCall(S, K, T, r, impli To find multiple roots, solve in different intervals: ```cs -using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; // Function with multiple roots: f(x) = sin(x) // Roots at x = 0, ±π, ±2π, ... @@ -332,7 +570,7 @@ foreach (var root in roots) Find critical points of a function by solving $f'(x) = 0$: ```cs -using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; // Find critical points of f(x) = x³ - 3x² + 2 Func f = x => x * x * x - 3 * x * x + 2; @@ -358,15 +596,34 @@ Console.WriteLine($"At x={cp2}: f''(x)={d2f(cp2):F1} → " + ## Choosing a Root Finding Method +### Method Comparison + +The following table provides a comprehensive comparison of all root-finding methods available in the ***Numerics*** library: + +| Feature | Bisection | Brent | Secant | Newton-Raphson | +|---------|-----------|-------|--------|----------------| +| **Convergence rate** | Linear $O(1/2^n)$ | Superlinear (adaptive) | ~1.618 (golden ratio) | Quadratic | +| **Requires derivative** | No | No | No | Yes | +| **Requires bracket** | Yes | Yes | No | No | +| **Guaranteed convergence** | Yes | Yes | No | No | +| **Function evals per step** | 1 | 1 | 1 | 2 ($f$ and $f'$) | +| **Typical iterations** | 30-50 | 5-10 | 5-10 | 4-6 | +| **Handles discontinuities** | Yes | Yes | Poorly | Poorly | +| **Risk of divergence** | None | None | Moderate | High | +| **Best for** | Proof of concept, robustness | General purpose (recommended) | When bracket unavailable | When $f'(x)$ is cheap and smooth | + +### Scenario-Based Recommendations + | Scenario | Recommended Method | Notes | |----------|-------------------|-------| -| General purpose | Brent | Best balance of speed and robustness | -| Have derivative | Newton-Raphson (robust version) | Fastest convergence | -| No derivative | Secant or Brent | Brent more robust, Secant faster | -| Difficult function | Bisection | Guaranteed convergence, but slow | -| Need absolute certainty | Bisection or Brent | Both guarantee convergence | -| Very smooth function | Newton-Raphson | Quadratic convergence | -| Poor initial guess | Brent or Robust Newton-Raphson | Fall back to bisection when needed | +| General purpose | **Brent** | Best balance of speed and robustness | +| Have analytical derivative | Newton-Raphson (robust version) | Fastest convergence with safety net | +| No derivative available | Brent | More robust than Secant | +| Difficult or ill-conditioned function | Bisection or Brent | Guaranteed convergence | +| Need absolute certainty | Brent | Guaranteed convergence with fast speed | +| Very smooth function with good guess | Newton-Raphson | Quadratic convergence | +| Poor or uncertain initial guess | Brent or Robust Newton-Raphson | Fall back to bisection when needed | +| Cannot bracket the root | Secant or Newton-Raphson | Consider `Brent.Bracket` to find bracket first | ## Convergence Criteria @@ -423,7 +680,7 @@ Both must be satisfied for convergence. The default tolerance is $10^{-8}$. ## Error Handling ```cs -using Numerics.Mathematics; +using Numerics.Mathematics.RootFinding; try { @@ -468,6 +725,10 @@ Newton-Raphson and Brent are typically fastest, while Bisection is slowest but m [2] Brent, R. P. (1973). *Algorithms for Minimization Without Derivatives*. Prentice-Hall, Englewood Cliffs, NJ. +[3] Sprott, J. C. (1991). *Numerical Recipes, Routines and Examples in Basic*. Cambridge University Press. + +[4] Süli, E. & Mayers, D. (2003). *An Introduction to Numerical Analysis*. Cambridge University Press. + --- [← Previous: Optimization](optimization.md) | [Back to Index](../index.md) | [Next: Linear Algebra →](linear-algebra.md) diff --git a/docs/mathematics/special-functions.md b/docs/mathematics/special-functions.md index 6ca62ad1..5b7a912c 100644 --- a/docs/mathematics/special-functions.md +++ b/docs/mathematics/special-functions.md @@ -2,11 +2,23 @@ [← Previous: Linear Algebra](linear-algebra.md) | [Back to Index](../index.md) | [Next: ODE Solvers →](ode-solvers.md) -The ***Numerics*** library provides essential special functions commonly used in statistical distributions, numerical analysis, and scientific computing. These include Gamma, Beta, Error functions, and combinatorial functions. +The ***Numerics*** library provides essential special functions commonly used in statistical distributions, numerical analysis, and scientific computing. These functions underpin many of the library's distribution and integration routines — for example, the Normal distribution CDF is computed using the error function, and the Chi-squared CDF uses the incomplete Gamma function. ## Gamma Function -The Gamma function Γ(x) extends the factorial function to real and complex numbers: Γ(n) = (n-1)! for positive integers. +The Gamma function is defined by the integral: + +```math +\Gamma(x) = \int_0^{\infty} t^{x-1} e^{-t} \, dt, \quad x > 0 +``` + +It extends the factorial function to real (and complex) numbers: $\Gamma(n) = (n-1)!$ for positive integers. The defining **recurrence relation** is: + +```math +\Gamma(x+1) = x \cdot \Gamma(x) +``` + +Two other important identities are the **reflection formula** $\Gamma(x)\Gamma(1-x) = \frac{\pi}{\sin(\pi x)}$ and the **duplication formula** $\Gamma(x)\Gamma(x + \frac{1}{2}) = \frac{\sqrt{\pi}}{2^{2x-1}}\Gamma(2x)$. The special value $\Gamma(\frac{1}{2}) = \sqrt{\pi}$ follows directly from the reflection formula. ### Basic Gamma Function @@ -24,13 +36,19 @@ Console.WriteLine($"Γ(3.5) = {g3:F3}"); // Verify: Γ(n+1) = n·Γ(n) double check = 3.5 * Gamma.Function(3.5); -Console.WriteLine($"4.5·Γ(3.5) = {check:F3}"); +Console.WriteLine($"3.5·Γ(3.5) = {check:F3}"); Console.WriteLine($"Γ(4.5) = {Gamma.Function(4.5):F3}"); ``` ### Log-Gamma Function -For large arguments, use log-gamma to avoid overflow: +For large arguments, $\Gamma(x)$ overflows double-precision floating point. The log-gamma function $\ln \Gamma(x)$ grows much more slowly and is used internally throughout the library for distribution computations. For large $x$, Stirling's approximation gives: + +```math +\ln \Gamma(x) \approx x \ln x - x + \frac{1}{2}\ln\!\left(\frac{2\pi}{x}\right) +``` + +Use log-gamma to avoid overflow: ```cs // Regular gamma would overflow for large x @@ -46,7 +64,7 @@ Console.WriteLine($"ln(Γ(200)) = {logGamma:F2}"); ### Digamma and Trigamma -Derivatives of the log-gamma function: +The digamma function $\psi(x) = \frac{d}{dx}\ln\Gamma(x) = \frac{\Gamma'(x)}{\Gamma(x)}$ and the trigamma function $\psi'(x) = \frac{d^2}{dx^2}\ln\Gamma(x)$ arise in maximum likelihood estimation for the Gamma, Beta, and Dirichlet distributions. The digamma function satisfies $\psi(x+1) = \psi(x) + \frac{1}{x}$ and for large $x$, $\psi(x) \approx \ln x - \frac{1}{2x}$. ```cs // Digamma: ψ(x) = d/dx[ln(Γ(x))] = Γ'(x)/Γ(x) @@ -63,31 +81,36 @@ Console.WriteLine($"ψ'(2) = {trigamma:F6}"); ### Incomplete Gamma Functions -Used in chi-squared and gamma distributions: +The **lower incomplete gamma function** is defined as the integral with a finite upper limit: + +```math +\gamma(a, x) = \int_0^{x} t^{a-1} e^{-t} \, dt +``` + +The library returns the **regularized** forms $P(a,x) = \gamma(a,x)/\Gamma(a)$ and $Q(a,x) = 1 - P(a,x)$, which range from 0 to 1. These are equivalent to the CDF and survival function of the Gamma distribution — the Chi-squared CDF with $k$ degrees of freedom is $P(k/2, x/2)$. ```cs -// Lower incomplete gamma: γ(a,x) = ∫₀ˣ t^(a-1)e^(-t) dt -double lowerIncomplete = Gamma.LowerIncomplete(a: 2.0, x: 3.0); +// Regularized lower incomplete gamma: P(a,x) = γ(a,x) / Γ(a) +double P = Gamma.LowerIncomplete(a: 2.0, x: 3.0); -// Upper incomplete gamma: Γ(a,x) = ∫ₓ^∞ t^(a-1)e^(-t) dt -double upperIncomplete = Gamma.UpperIncomplete(a: 2.0, x: 3.0); +// Regularized upper incomplete gamma: Q(a,x) = Γ(a,x) / Γ(a) +double Q = Gamma.UpperIncomplete(a: 2.0, x: 3.0); -// Verify: γ(a,x) + Γ(a,x) = Γ(a) -double sum = lowerIncomplete + upperIncomplete; -double gamma = Gamma.Function(2.0); +// Verify: P(a,x) + Q(a,x) = 1 +double sum = P + Q; -Console.WriteLine($"Lower: {lowerIncomplete:F6}"); -Console.WriteLine($"Upper: {upperIncomplete:F6}"); -Console.WriteLine($"Sum: {sum:F6}, Γ(2): {gamma:F6}"); +Console.WriteLine($"P(2, 3): {P:F6}"); +Console.WriteLine($"Q(2, 3): {Q:F6}"); +Console.WriteLine($"P + Q: {sum:F6}"); // 1.000000 ``` -### Regularized Incomplete Gamma +### Alternative: Gamma.Incomplete -Normalized version: P(a,x) = γ(a,x)/Γ(a) +An alternative method with different parameter ordering: ```cs // Regularized incomplete gamma (CDF of Gamma distribution) -double P = Gamma.Incomplete(x: 3.0, alpha: 2.0); +double P = Gamma.Incomplete(X: 3.0, alpha: 2.0); Console.WriteLine($"P(2, 3) = {P:F6}"); Console.WriteLine("This equals the Gamma(2,1) CDF at x=3"); @@ -99,7 +122,13 @@ Console.WriteLine($"P(2, {xInv:F3}) = 0.9"); ## Beta Function -The Beta function relates to the Gamma function: B(a,b) = Γ(a)Γ(b)/Γ(a+b) +The Beta function is defined by the integral: + +```math +B(a,b) = \int_0^1 t^{a-1}(1-t)^{b-1}\,dt = \frac{\Gamma(a)\Gamma(b)}{\Gamma(a+b)} +``` + +The relationship to the Gamma function makes computation straightforward via $\ln B(a,b) = \ln\Gamma(a) + \ln\Gamma(b) - \ln\Gamma(a+b)$. ### Basic Beta Function @@ -121,18 +150,24 @@ Console.WriteLine($"Γ(2)Γ(3)/Γ(5) = {betaCheck:F6}"); ### Incomplete Beta Function -Used in Beta distribution and Student's t-test: +The incomplete Beta function is defined as: + +```math +B_x(a,b) = \int_0^x t^{a-1}(1-t)^{b-1}\,dt +``` + +The library returns the **regularized** form $I_x(a,b) = B_x(a,b) / B(a,b)$, which ranges from 0 to 1 and is the CDF of the Beta distribution. It is also used internally for the Student's t-test, the F-distribution CDF, and the binomial distribution CDF. ```cs -// Incomplete beta: Bₓ(a,b) = ∫₀ˣ t^(a-1)(1-t)^(b-1) dt -double incompleteBeta = Beta.Incomplete(a: 2.0, b: 3.0, x: 0.4); +// Regularized incomplete beta: Iₓ(a,b) = Bₓ(a,b) / B(a,b) +double Ix = Beta.Incomplete(a: 2.0, b: 3.0, x: 0.4); -Console.WriteLine($"Bₓ(2,3,0.4) = {incompleteBeta:F6}"); -Console.WriteLine("This equals integral from 0 to 0.4"); +Console.WriteLine($"Iₓ(2,3,0.4) = {Ix:F6}"); +Console.WriteLine("This is the CDF of Beta(2,3) at x=0.4"); -// Regularized incomplete beta (CDF of Beta distribution) -double I = incompleteBeta / Beta.Function(2.0, 3.0); -Console.WriteLine($"I(2,3,0.4) = {I:F6}"); +// To recover the raw (non-regularized) incomplete beta: +double rawIncomplete = Ix * Beta.Function(2.0, 3.0); +Console.WriteLine($"Bₓ(2,3,0.4) = {rawIncomplete:F6}"); ``` ### Inverse Incomplete Beta @@ -152,7 +187,13 @@ Console.WriteLine($"Verification: I(2,3,{x:F4}) = {check:F6}"); ## Error Function -The error function is the integral of the Gaussian distribution: +The error function is defined as the integral: + +```math +\text{erf}(x) = \frac{2}{\sqrt{\pi}} \int_0^x e^{-t^2}\,dt +``` + +It is closely related to the Normal distribution CDF: $\Phi(x) = \frac{1}{2}\left[1 + \text{erf}\!\left(\frac{x}{\sqrt{2}}\right)\right]$. The library uses this relationship internally to compute the standard normal CDF. The complementary error function $\text{erfc}(x) = 1 - \text{erf}(x)$ is computed directly (rather than via subtraction) for numerical accuracy when $x$ is large, where $\text{erf}(x) \approx 1$ and the subtraction $1 - \text{erf}(x)$ would lose precision. ### Error Function and Complement @@ -250,6 +291,73 @@ int count = combinations.Count(); Console.WriteLine($"Total: {count} combinations"); ``` +## Bessel Functions + +Bessel functions arise in problems with cylindrical symmetry and in directional statistics (e.g., the Von Mises distribution). The library provides modified Bessel functions of the first kind. + +### Modified Bessel Functions of the First Kind + +```cs +using Numerics.Mathematics.SpecialFunctions; + +// I₀(x) - Modified Bessel function of the first kind, order 0 +double i0 = Bessel.I0(2.5); +Console.WriteLine($"I₀(2.5) = {i0:F6}"); // ≈ 3.289839 + +// I₁(x) - Modified Bessel function of the first kind, order 1 +double i1 = Bessel.I1(2.5); +Console.WriteLine($"I₁(2.5) = {i1:F6}"); // ≈ 2.516716 + +// Iₙ(x) - Modified Bessel function of the first kind, integer order +double i_n = Bessel.In(2, 3.0); // I₂(3.0) +Console.WriteLine($"I₂(3.0) = {i_n:F6}"); +``` + +### Log-Space Bessel Computations + +For large arguments where the Bessel function overflows, compute in log space manually: + +```cs +// Log I₀(x) - compute manually to avoid overflow for large x +double x = 500.0; +double logI0 = Math.Log(Bessel.I0(x)); +Console.WriteLine($"ln(I₀({x})) = {logI0:F4}"); + +// Log I₁(x) +double logI1 = Math.Log(Bessel.I1(x)); +Console.WriteLine($"ln(I₁({x})) = {logI1:F4}"); +``` + +### Bessel Function Ratios + +Ratios of Bessel functions appear in maximum likelihood estimation for the Von Mises distribution: + +```cs +// I₁(x)/I₀(x) ratio - used in Von Mises MLE +double ratio = Bessel.I1(5.0) / Bessel.I0(5.0); +Console.WriteLine($"I₁(5)/I₀(5) = {ratio:F6}"); + +// For large kappa, this ratio approaches 1 +double ratioLarge = Bessel.I1(100.0) / Bessel.I0(100.0); +Console.WriteLine($"I₁(100)/I₀(100) = {ratioLarge:F8}"); +``` + +### Modified Bessel Functions of the Second Kind + +```cs +// K₀(x) - Modified Bessel function of the second kind, order 0 +double k0 = Bessel.K0(1.0); +Console.WriteLine($"K₀(1.0) = {k0:F6}"); // ≈ 0.421024 + +// K₁(x) - Order 1 +double k1 = Bessel.K1(1.0); +Console.WriteLine($"K₁(1.0) = {k1:F6}"); // ≈ 0.601907 + +// Kₙ(x) - Integer order +double k_n = Bessel.Kn(2, 1.5); +Console.WriteLine($"K₂(1.5) = {k_n:F6}"); +``` + ## Practical Applications ### Example 1: Gamma Distribution Moments @@ -308,7 +416,7 @@ int k = 5; // Degrees of freedom double x = 8.0; // CDF at x -double cdf = Gamma.Incomplete(x: x / 2.0, alpha: k / 2.0); +double cdf = Gamma.Incomplete(X: x / 2.0, alpha: k / 2.0); Console.WriteLine($"Chi-squared({k}) CDF at {x}:"); Console.WriteLine($" P(X ≤ {x}) = {cdf:F6}"); @@ -373,15 +481,26 @@ Console.WriteLine($" Relative error = {relativeError:P4}"); | **Gamma** | Factorial extension | `Function()`, `LogGamma()`, `Digamma()` | | **Incomplete Gamma** | Chi-squared, Gamma CDF | `LowerIncomplete()`, `UpperIncomplete()` | | **Beta** | Beta distribution | `Function()`, `Incomplete()` | -| **Error** | Normal distribution | `Function()`, `Erfc()`, `InverseErf()` | +| **Error** | Normal distribution | `Function()`, `Erfc()`, `InverseErf()`, `InverseErfc()` | | **Factorial** | Combinatorics | `Function()`, `BinomialCoefficient()` | +| **Bessel** | Cylindrical, directional stats | `I0()`, `I1()`, `In()`, `K0()`, `K1()`, `Kn()`, `J0()`, `J1()`, `Jn()`, `Y0()`, `Y1()`, `Yn()` | ## Implementation Notes -- All functions use high-precision approximations +- All functions use high-precision polynomial or rational approximations - Log-space variants prevent overflow for large arguments -- Inverse functions use Newton-Raphson iteration -- Special care for edge cases and numerical stability +- Inverse functions use Newton-Raphson iteration with appropriate starting values +- Special care for edge cases: $\Gamma(x)$ near poles at non-positive integers, $\text{erf}(x)$ for large $|x|$, incomplete gamma/beta for extreme parameter ratios + +--- + +## References + +[1] M. Abramowitz and I. A. Stegun, *Handbook of Mathematical Functions*, New York: Dover Publications, 1964. + +[2] W. H. Press, S. A. Teukolsky, W. T. Vetterling and B. P. Flannery, *Numerical Recipes: The Art of Scientific Computing*, 3rd ed., Cambridge, UK: Cambridge University Press, 2007. + +[3] NIST Digital Library of Mathematical Functions, https://dlmf.nist.gov/. --- diff --git a/docs/references.md b/docs/references.md index 1a586bbb..001a8ac4 100644 --- a/docs/references.md +++ b/docs/references.md @@ -28,176 +28,262 @@ This is a consolidated bibliography of all references cited throughout the ***Nu [9] Wilks, D. S. (2011). *Statistical Methods in the Atmospheric Sciences* (3rd ed.). Academic Press. +[10] Fisher, R. A. (1930). The moments of the distribution for normal samples of measures of departure from normality. *Proceedings of the Royal Society of London. Series A*, 130(812), 16-28. + +[11] Welford, B. P. (1962). Note on a method for calculating corrected sums of squares and products. *Technometrics*, 4(3), 419-420. + +[12] Kendall, M. G. (1938). A new measure of rank correlation. *Biometrika*, 30(1/2), 81-93. + +[13] Mood, A. M., Graybill, F. A., & Boes, D. C. (1974). *Introduction to the Theory of Statistics* (3rd ed.). McGraw-Hill. + +[14] Casella, G., & Berger, R. L. (2002). *Statistical Inference* (2nd ed.). Duxbury/Thomson. + --- ## Hydrology and Water Resources -[10] England, J. F., et al. (2019). Guidelines for Determining Flood Flow Frequency—Bulletin 17C. *U.S. Geological Survey Techniques and Methods*, Book 4, Chapter B5. +[15] England, J. F., et al. (2018). Guidelines for Determining Flood Flow Frequency—Bulletin 17C. *U.S. Geological Survey Techniques and Methods*, Book 4, Chapter B5. + +[16] Stedinger, J. R., Vogel, R. M., & Foufoula-Georgiou, E. (1993). Frequency analysis of extreme events. In D. R. Maidment (Ed.), *Handbook of Hydrology* (Chapter 18). McGraw-Hill. -[11] Stedinger, J. R., Vogel, R. M., & Foufoula-Georgiou, E. (1993). Frequency analysis of extreme events. In D. R. Maidment (Ed.), *Handbook of Hydrology* (Chapter 18). McGraw-Hill. +[17] Helsel, D. R., Hirsch, R. M., Ryberg, K. R., Archfield, S. A., & Gilroy, E. J. (2020). *Statistical Methods in Water Resources*. U.S. Geological Survey Techniques and Methods, Book 4, Chapter A3. -[12] Helsel, D. R., Hirsch, R. M., Ryberg, K. R., Archfield, S. A., & Gilroy, E. J. (2020). *Statistical Methods in Water Resources*. U.S. Geological Survey Techniques and Methods, Book 4, Chapter A3. +[18] Cunnane, C. (1978). Unbiased plotting positions—A review. *Journal of Hydrology*, 37(3-4), 205-222. -[13] Cunnane, C. (1978). Unbiased plotting positions—A review. *Journal of Hydrology*, 37(3-4), 205-222. +[19] Cohn, T. A., England, J. F., Berenbrock, C. E., Mason, R. R., Stedinger, J. R., & Lamontagne, J. R. (2013). A generalized Grubbs-Beck test statistic for detecting multiple potentially influential low outliers in flood series. *Water Resources Research*, 49(8), 5047-5058. -[14] Cohn, T. A., England, J. F., Berenbrock, C. E., Mason, R. R., Stedinger, J. R., & Lamontagne, J. R. (2013). A generalized Grubbs-Beck test statistic for detecting multiple potentially influential low outliers in flood series. *Water Resources Research*, 49(8), 5047-5058. +[20] Eckhardt, K. (2005). How to construct recursive digital filters for baseflow separation. *Hydrological Processes*, 19(2), 507-515. -[15] Eckhardt, K. (2005). How to construct recursive digital filters for baseflow separation. *Hydrological Processes*, 19(2), 507-515. +[21] Rao, A. R., & Hamed, K. H. (2000). *Flood Frequency Analysis*. CRC Press. --- ## Model Evaluation -[16] Moriasi, D. N., Arnold, J. G., Van Liew, M. W., Bingner, R. L., Harmel, R. D., & Veith, T. L. (2007). Model evaluation guidelines for systematic quantification of accuracy in watershed simulations. *Transactions of the ASABE*, 50(3), 885-900. +[22] Moriasi, D. N., Arnold, J. G., Van Liew, M. W., Bingner, R. L., Harmel, R. D., & Veith, T. L. (2007). Model evaluation guidelines for systematic quantification of accuracy in watershed simulations. *Transactions of the ASABE*, 50(3), 885-900. -[17] Moriasi, D. N., Gitau, M. W., Pai, N., & Daggupati, P. (2015). Hydrologic and water quality models: Performance measures and evaluation criteria. *Transactions of the ASABE*, 58(6), 1763-1785. +[23] Moriasi, D. N., Gitau, M. W., Pai, N., & Daggupati, P. (2015). Hydrologic and water quality models: Performance measures and evaluation criteria. *Transactions of the ASABE*, 58(6), 1763-1785. -[18] Nash, J. E., & Sutcliffe, J. V. (1970). River flow forecasting through conceptual models part I—A discussion of principles. *Journal of Hydrology*, 10(3), 282-290. +[24] Nash, J. E., & Sutcliffe, J. V. (1970). River flow forecasting through conceptual models part I—A discussion of principles. *Journal of Hydrology*, 10(3), 282-290. -[19] Gupta, H. V., Kling, H., Yilmaz, K. K., & Martinez, G. F. (2009). Decomposition of the mean squared error and NSE performance criteria: Implications for improving hydrological modelling. *Journal of Hydrology*, 377(1-2), 80-91. +[25] Gupta, H. V., Kling, H., Yilmaz, K. K., & Martinez, G. F. (2009). Decomposition of the mean squared error and NSE performance criteria: Implications for improving hydrological modelling. *Journal of Hydrology*, 377(1-2), 80-91. -[20] Legates, D. R., & McCabe, G. J. (1999). Evaluating the use of "goodness-of-fit" measures in hydrologic and hydroclimatic model validation. *Water Resources Research*, 35(1), 233-241. +[26] Legates, D. R., & McCabe, G. J. (1999). Evaluating the use of "goodness-of-fit" measures in hydrologic and hydroclimatic model validation. *Water Resources Research*, 35(1), 233-241. -[21] Burnham, K. P., & Anderson, D. R. (2002). *Model Selection and Multimodel Inference: A Practical Information-Theoretic Approach* (2nd ed.). Springer. +[27] Burnham, K. P., & Anderson, D. R. (2002). *Model Selection and Multimodel Inference: A Practical Information-Theoretic Approach* (2nd ed.). Springer. + +[28] Akaike, H. (1974). A new look at the statistical model identification. *IEEE Transactions on Automatic Control*, 19(6), 716-723. + +[29] Schwarz, G. (1978). Estimating the dimension of a model. *Annals of Statistics*, 6(2), 461-464. + +[30] Murphy, A. H. (1988). Skill scores based on the mean square error and their relationships to the correlation coefficient. *Monthly Weather Review*, 116(12), 2417-2424. + +[31] Kling, H., Fuchs, M., & Paulin, M. (2012). Runoff conditions in the upper Danube basin under an ensemble of climate change scenarios. *Journal of Hydrology*, 424-425, 264-277. --- ## Copulas and Multivariate Analysis -[22] Nelsen, R. B. (2006). *An Introduction to Copulas* (2nd ed.). Springer. +[32] Nelsen, R. B. (2006). *An Introduction to Copulas* (2nd ed.). Springer. + +[33] Joe, H. (1997). *Multivariate Models and Dependence Concepts*. Chapman & Hall. -[23] Joe, H. (1997). *Multivariate Models and Dependence Concepts*. Chapman & Hall. +[34] Genest, C., & Favre, A.-C. (2007). Everything you always wanted to know about copula modeling but were afraid to ask. *Journal of Hydrologic Engineering*, 12(4), 347-368. -[24] Genest, C., & Favre, A.-C. (2007). Everything you always wanted to know about copula modeling but were afraid to ask. *Journal of Hydrologic Engineering*, 12(4), 347-368. +[35] Salvadori, G., De Michele, C., Kottegoda, N. T., & Rosso, R. (2007). *Extremes in Nature: An Approach Using Copulas*. Springer. -[25] Salvadori, G., De Michele, C., Kottegoda, N. T., & Rosso, R. (2007). *Extremes in Nature: An Approach Using Copulas*. Springer. +[36] Salvadori, G., & De Michele, C. (2004). Frequency analysis via copulas: Theoretical aspects and applications to hydrological events. *Water Resources Research*, 40(12). -[26] Salvadori, G., & De Michele, C. (2004). Frequency analysis via copulas: Theoretical aspects and applications to hydrological events. *Water Resources Research*, 40(12). +[37] Anderson, T. W. (2003). *An Introduction to Multivariate Statistical Analysis* (3rd ed.). Wiley. + +[38] Kotz, S., Balakrishnan, N., & Johnson, N. L. (2000). *Continuous Multivariate Distributions, Volume 1: Models and Applications* (2nd ed.). Wiley. + +[39] Johnson, N. L., Kotz, S., & Balakrishnan, N. (1997). *Discrete Multivariate Distributions*. Wiley. + +[40] Tong, Y. L. (2012). *The Multivariate Normal Distribution*. Springer Science & Business Media. + +[41] Kotz, S., & Nadarajah, S. (2004). *Multivariate t Distributions and Their Applications*. Cambridge University Press. --- ## Numerical Integration -[27] Piessens, R., de Doncker-Kapenga, E., Überhuber, C. W., & Kahaner, D. K. (1983). *QUADPACK: A Subroutine Package for Automatic Integration*. Springer. +[42] Piessens, R., de Doncker-Kapenga, E., Überhuber, C. W., & Kahaner, D. K. (1983). *QUADPACK: A Subroutine Package for Automatic Integration*. Springer. -[28] Press, W. H., & Farrar, G. R. (1990). Recursive stratified sampling for multidimensional Monte Carlo integration. *Computers in Physics*, 4(2), 190-195. +[43] Press, W. H., & Farrar, G. R. (1990). Recursive stratified sampling for multidimensional Monte Carlo integration. *Computers in Physics*, 4(2), 190-195. -[29] Lepage, G. P. (1978). A new algorithm for adaptive multidimensional integration. *Journal of Computational Physics*, 27(2), 192-203. +[44] Lepage, G. P. (1978). A new algorithm for adaptive multidimensional integration. *Journal of Computational Physics*, 27(2), 192-203. --- ## Numerical Differentiation -[30] Ridders, C. J. F. (1982). Accurate computation of F'(x) and F'(x)F''(x). *Advances in Engineering Software*, 4(2), 75-76. +[45] Ridders, C. J. F. (1982). Accurate computation of F'(x) and F'(x)F''(x). *Advances in Engineering Software*, 4(2), 75-76. --- ## Optimization -[31] Nocedal, J., & Wright, S. J. (2006). *Numerical Optimization* (2nd ed.). Springer. +[46] Nocedal, J., & Wright, S. J. (2006). *Numerical Optimization* (2nd ed.). Springer. -[32] Nelder, J. A., & Mead, R. (1965). A simplex method for function minimization. *The Computer Journal*, 7(4), 308-313. +[47] Nelder, J. A., & Mead, R. (1965). A simplex method for function minimization. *The Computer Journal*, 7(4), 308-313. -[33] Storn, R., & Price, K. (1997). Differential evolution—A simple and efficient heuristic for global optimization over continuous spaces. *Journal of Global Optimization*, 11(4), 341-359. +[48] Storn, R., & Price, K. (1997). Differential evolution—A simple and efficient heuristic for global optimization over continuous spaces. *Journal of Global Optimization*, 11(4), 341-359. -[34] Duan, Q., Sorooshian, S., & Gupta, V. K. (1994). Optimal use of the SCE-UA global optimization method for calibrating watershed models. *Journal of Hydrology*, 158(3-4), 265-284. +[49] Duan, Q., Sorooshian, S., & Gupta, V. K. (1994). Optimal use of the SCE-UA global optimization method for calibrating watershed models. *Journal of Hydrology*, 158(3-4), 265-284. -[35] Kennedy, J., & Eberhart, R. (1995). Particle swarm optimization. *Proceedings of ICNN'95*, 4, 1942-1948. +[50] Kennedy, J., & Eberhart, R. (1995). Particle swarm optimization. *Proceedings of ICNN'95*, 4, 1942-1948. --- ## Linear Algebra -[36] Golub, G. H., & Van Loan, C. F. (2013). *Matrix Computations* (4th ed.). Johns Hopkins University Press. +[51] Golub, G. H., & Van Loan, C. F. (2013). *Matrix Computations* (4th ed.). Johns Hopkins University Press. -[37] Trefethen, L. N., & Bau, D. (1997). *Numerical Linear Algebra*. SIAM. +[52] Trefethen, L. N., & Bau, D. (1997). *Numerical Linear Algebra*. SIAM. --- ## Root Finding -[38] Brent, R. P. (1973). *Algorithms for Minimization without Derivatives*. Prentice-Hall. +[53] Brent, R. P. (1973). *Algorithms for Minimization without Derivatives*. Prentice-Hall. + +[54] Sprott, J. C. (1991). *Numerical Recipes, Routines and Examples in Basic*. Cambridge University Press. + +[55] Süli, E., & Mayers, D. (2003). *An Introduction to Numerical Analysis*. Cambridge University Press. --- ## Interpolation -[39] Akima, H. (1970). A new method of interpolation and smooth curve fitting based on local procedures. *Journal of the ACM*, 17(4), 589-602. +[56] Akima, H. (1970). A new method of interpolation and smooth curve fitting based on local procedures. *Journal of the ACM*, 17(4), 589-602. -[40] de Boor, C. (2001). *A Practical Guide to Splines* (Rev. ed.). Springer. +[57] de Boor, C. (2001). *A Practical Guide to Splines* (Rev. ed.). Springer. --- ## Random Number Generation -[41] Matsumoto, M., & Nishimura, T. (1998). Mersenne Twister: A 623-dimensionally equidistributed uniform pseudo-random number generator. *ACM Transactions on Modeling and Computer Simulation*, 8(1), 3-30. +[58] Matsumoto, M., & Nishimura, T. (1998). Mersenne Twister: A 623-dimensionally equidistributed uniform pseudo-random number generator. *ACM Transactions on Modeling and Computer Simulation*, 8(1), 3-30. -[42] Niederreiter, H. (1992). *Random Number Generation and Quasi-Monte Carlo Methods*. SIAM. +[59] Niederreiter, H. (1992). *Random Number Generation and Quasi-Monte Carlo Methods*. SIAM. -[43] McKay, M. D., Beckman, R. J., & Conover, W. J. (1979). A comparison of three methods for selecting values of input variables in the analysis of output from a computer code. *Technometrics*, 21(2), 239-245. +[60] McKay, M. D., Beckman, R. J., & Conover, W. J. (1979). A comparison of three methods for selecting values of input variables in the analysis of output from a computer code. *Technometrics*, 21(2), 239-245. -[44] Owen, A. B. (2003). Quasi-Monte Carlo sampling. In *Monte Carlo Ray Tracing: Siggraph 2003 Course 44*, 69-88. +[61] Owen, A. B. (2003). Quasi-Monte Carlo sampling. In *Monte Carlo Ray Tracing: Siggraph 2003 Course 44*, 69-88. --- ## MCMC and Bayesian Methods -[45] Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). *Bayesian Data Analysis* (3rd ed.). CRC Press. +[62] Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). *Bayesian Data Analysis* (3rd ed.). CRC Press. + +[63] Robert, C. P., & Casella, G. (2004). *Monte Carlo Statistical Methods* (2nd ed.). Springer. + +[64] Haario, H., Saksman, E., & Tamminen, J. (2001). An adaptive Metropolis algorithm. *Bernoulli*, 7(2), 223-242. + +[65] ter Braak, C. J. F., & Vrugt, J. A. (2008). Differential Evolution Markov Chain with snooker updater and fewer chains. *Statistics and Computing*, 18(4), 435-446. + +[66] Neal, R. M. (2011). MCMC using Hamiltonian dynamics. In *Handbook of Markov Chain Monte Carlo* (pp. 113-162). CRC Press. + +[67] Vehtari, A., Gelman, A., Simpson, D., Carpenter, B., & Bürkner, P.-C. (2021). Rank-normalization, folding, and localization: An improved R-hat for assessing convergence of MCMC. *Bayesian Analysis*, 16(2), 667-718. -[46] Robert, C. P., & Casella, G. (2004). *Monte Carlo Statistical Methods* (2nd ed.). Springer. +[68] Vrugt, J. A. (2016). Markov chain Monte Carlo simulation using the DREAM software package: Theory, concepts, and MATLAB implementation. *Environmental Modelling & Software*, 75, 273-316. -[47] Haario, H., Saksman, E., & Tamminen, J. (2001). An adaptive Metropolis algorithm. *Bernoulli*, 7(2), 223-242. +[69] Hoffman, M. D., & Gelman, A. (2014). The No-U-Turn Sampler: Adaptively setting path lengths in Hamiltonian Monte Carlo. *Journal of Machine Learning Research*, 15(47), 1593-1623. -[48] ter Braak, C. J. F., & Vrugt, J. A. (2008). Differential Evolution Markov Chain with snooker updater and fewer chains. *Statistics and Computing*, 18(4), 435-446. +[70] Metropolis, N., Rosenbluth, A. W., Rosenbluth, M. N., Teller, A. H., & Teller, E. (1953). Equation of state calculations by fast computing machines. *The Journal of Chemical Physics*, 21(6), 1087-1092. -[49] Neal, R. M. (2011). MCMC using Hamiltonian dynamics. In *Handbook of Markov Chain Monte Carlo* (pp. 113-162). CRC Press. +[71] Gelman, A., & Rubin, D. B. (1992). Inference from iterative simulation using multiple sequences. *Statistical Science*, 7(4), 457-472. -[50] Vehtari, A., Gelman, A., Simpson, D., Carpenter, B., & Bürkner, P. C. (2021). Rank-normalization, folding, and localization: An improved R-hat for assessing convergence of MCMC. *Bayesian Analysis*, 16(2), 667-718. +[72] Sobol, I. M. (1967). On the distribution of points in a cube and the approximate evaluation of integrals. *USSR Computational Mathematics and Mathematical Physics*, 7(4), 86-112. -[51] Vrugt, J. A. (2016). Markov chain Monte Carlo simulation using the DREAM software package: Theory, concepts, and MATLAB implementation. *Environmental Modelling & Software*, 75, 273-316. +[73] Roberts, G. O., Gelman, A., & Gilks, W. R. (1997). Weak convergence and optimal scaling of random walk Metropolis algorithms. *Annals of Applied Probability*, 7(1), 110-120. + +[74] Roberts, G. O., & Rosenthal, J. S. (2001). Optimal scaling for various Metropolis-Hastings algorithms. *Statistical Science*, 16(4), 351-367. + +[75] Betancourt, M. (2017). A conceptual introduction to Hamiltonian Monte Carlo. *arXiv preprint arXiv:1701.02434*. + +[76] Geyer, C. J. (1992). Practical Markov chain Monte Carlo. *Statistical Science*, 7(4), 473-483. + +[77] Flegal, J. M., Haran, M., & Jones, G. L. (2008). Markov chain Monte Carlo: Can we trust the third significant figure? *Statistical Science*, 23(2), 250-260. --- ## Uncertainty Analysis -[52] Efron, B., & Tibshirani, R. J. (1993). *An Introduction to the Bootstrap*. Chapman & Hall. +[78] Efron, B., & Tibshirani, R. J. (1993). *An Introduction to the Bootstrap*. Chapman & Hall. + +[79] Stedinger, J. R. (1983). Confidence intervals for design events. *Journal of Hydraulic Engineering*, 109(1), 13-27. + +[80] Hirsch, R. M., & Stedinger, J. R. (1987). Plotting positions for historical floods and their precision. *Water Resources Research*, 23(4), 715-727. -[53] Stedinger, J. R. (1983). Confidence intervals for design events. *Journal of Hydraulic Engineering*, 109(1), 13-27. +[81] Efron, B. (1987). Better bootstrap confidence intervals. *Journal of the American Statistical Association*, 82(397), 171-185. -[54] Hirsch, R. M., & Stedinger, J. R. (1987). Plotting positions for historical floods and their precision. *Water Resources Research*, 23(4), 715-727. +[82] Davison, A. C., & Hinkley, D. V. (1997). *Bootstrap Methods and Their Application*. Cambridge University Press. --- ## Special Distributions -[55] Weibull, W. (1951). A statistical distribution function of wide applicability. *Journal of Applied Mechanics*, 18(3), 293-297. +[83] Weibull, W. (1951). A statistical distribution function of wide applicability. *Journal of Applied Mechanics*, 18(3), 293-297. -[56] Vose, D. (2008). *Risk Analysis: A Quantitative Guide* (3rd ed.). Wiley. +[84] Vose, D. (2008). *Risk Analysis: A Quantitative Guide* (3rd ed.). Wiley. -[57] McLachlan, G., & Peel, D. (2000). *Finite Mixture Models*. Wiley. +[85] McLachlan, G., & Peel, D. (2000). *Finite Mixture Models*. Wiley. -[58] Silverman, B. W. (1986). *Density Estimation for Statistics and Data Analysis*. Chapman & Hall. +[86] Silverman, B. W. (1986). *Density Estimation for Statistics and Data Analysis*. Chapman & Hall. + +[87] Mardia, K. V., & Jupp, P. E. (2000). *Directional Statistics*. Wiley. --- ## Goodness-of-Fit Tests -[59] D'Agostino, R. B., & Stephens, M. A. (1986). *Goodness-of-Fit Techniques*. Marcel Dekker. +[88] D'Agostino, R. B., & Stephens, M. A. (1986). *Goodness-of-Fit Techniques*. Marcel Dekker. + +[89] Anderson, T. W., & Darling, D. A. (1954). A test of goodness of fit. *Journal of the American Statistical Association*, 49(268), 765-769. --- ## Time Series -[60] Box, G. E. P., Jenkins, G. M., Reinsel, G. C., & Ljung, G. M. (2015). *Time Series Analysis: Forecasting and Control* (5th ed.). Wiley. +[90] Box, G. E. P., Jenkins, G. M., Reinsel, G. C., & Ljung, G. M. (2015). *Time Series Analysis: Forecasting and Control* (5th ed.). Wiley. -[61] Box, G. E. P., & Cox, D. R. (1964). An analysis of transformations. *Journal of the Royal Statistical Society: Series B*, 26(2), 211-252. +[91] Box, G. E. P., & Cox, D. R. (1964). An analysis of transformations. *Journal of the Royal Statistical Society: Series B*, 26(2), 211-252. --- ## Data Sources -[62] U.S. Geological Survey. *USGS Water Services*. https://waterservices.usgs.gov/ +[92] U.S. Geological Survey. *USGS Water Services*. https://waterservices.usgs.gov/ + +[93] Environment and Climate Change Canada. *Historical Hydrometric Data*. https://wateroffice.ec.gc.ca/ + +[94] Australian Bureau of Meteorology. *Water Data Online*. https://www.bom.gov.au/waterdata/ + +--- + +## Machine Learning + +[95] Nelder, J. A., & Wedderburn, R. W. M. (1972). Generalized linear models. *Journal of the Royal Statistical Society: Series A*, 135(3), 370-384. + +[96] Breiman, L., Friedman, J. H., Olshen, R. A., & Stone, C. J. (1984). *Classification and Regression Trees*. Wadsworth. + +[97] Breiman, L. (2001). Random forests. *Machine Learning*, 45(1), 5-32. + +[98] Cover, T., & Hart, P. (1967). Nearest neighbor pattern classification. *IEEE Transactions on Information Theory*, 13(1), 21-27. + +[99] Zhang, H. (2004). The optimality of naive Bayes. *Proceedings of the Seventeenth International FLAIRS Conference*, 562-567. + +[100] MacQueen, J. (1967). Some methods for classification and analysis of multivariate observations. *Proceedings of the Fifth Berkeley Symposium on Mathematical Statistics and Probability*, 1, 281-297. + +[101] Bishop, C. M. (2006). *Pattern Recognition and Machine Learning*. Springer. + +[102] Jenks, G. F. (1967). The data model concept in statistical mapping. *International Yearbook of Cartography*, 7, 186-190. + +[103] Hastie, T., Tibshirani, R., & Friedman, J. (2009). *The Elements of Statistical Learning* (2nd ed.). Springer. -[63] Environment and Climate Change Canada. *Historical Hydrometric Data*. https://wateroffice.ec.gc.ca/ +[104] Dempster, A. P., Laird, N. M., & Rubin, D. B. (1977). Maximum likelihood from incomplete data via the EM algorithm. *Journal of the Royal Statistical Society: Series B*, 39(1), 1-38. -[64] Australian Bureau of Meteorology. *Water Data Online*. http://www.bom.gov.au/waterdata/ +[105] Arthur, D., & Vassilvitskii, S. (2007). k-means++: The advantages of careful seeding. *Proceedings of the 18th Annual ACM-SIAM Symposium on Discrete Algorithms*, 1027-1035. diff --git a/docs/sampling/convergence-diagnostics.md b/docs/sampling/convergence-diagnostics.md index dad65a55..f38fa337 100644 --- a/docs/sampling/convergence-diagnostics.md +++ b/docs/sampling/convergence-diagnostics.md @@ -17,6 +17,20 @@ MCMC samplers: - How many independent samples do we have? - Is the warmup period sufficient? +### Theoretical Foundation + +The theoretical justification for MCMC rests on the **ergodic theorem**: under regularity conditions (irreducibility, aperiodicity, positive recurrence), the time average of a function along the Markov chain converges to its expectation under the stationary distribution: + +```math +\frac{1}{N}\sum_{t=1}^{N}f(\theta_t) \;\xrightarrow{\;a.s.\;}\; \mathbb{E}_\pi[f(\theta)] \quad \text{as } N \to \infty +``` + +where $\pi$ is the target (posterior) distribution. This guarantee is asymptotic -- for any finite $N$, we need diagnostics to assess whether we are close enough to this limit. + +**Burn-in (warmup)** refers to the initial transient phase where the chain has not yet reached the stationary distribution. Samples from this phase are drawn from a distribution that depends on the arbitrary starting point, not from $\pi$. Discarding these samples is essential for valid inference. + +**Mixing** describes how quickly the chain "forgets" its current state and explores the full support of $\pi$. A well-mixing chain has rapidly decaying autocorrelation -- the correlation between $\theta_t$ and $\theta_{t+k}$ diminishes quickly with lag $k$. Poorly mixing chains remain in local regions for long periods, producing highly correlated samples and requiring far more iterations to achieve reliable estimates. + ## Gelman-Rubin Statistic (R̂) The Gelman-Rubin diagnostic compares within-chain and between-chain variance [[1]](#1). Values near 1.0 indicate convergence. @@ -34,10 +48,8 @@ sampler.WarmupIterations = 2000; sampler.Iterations = 5000; sampler.Sample(); -// Get chains -var chains = new List>(); -// Extract chains from sampler output -// (Implementation depends on sampler structure) +// Get chains from sampler +var chains = sampler.MarkovChains.ToList(); // Compute Gelman-Rubin for each parameter int warmup = sampler.WarmupIterations; @@ -67,15 +79,61 @@ for (int i = 0; i < rHat.Length; i++) | R̂ ≥ 1.2 | Poor convergence | Investigate | **Formula:** + +```math +\hat{R} = \sqrt{\frac{\hat{V}}{W}} +``` +Where $W$ is the mean within-chain variance, $B$ is the between-chain variance, and: +```math +\hat{V} = \frac{n-1}{n}W + \frac{1}{n}B ``` -R̂ = √(Var_total / W) -Where: -- W = within-chain variance (average) -- B = between-chain variance -- Var_total = ((n-1)/n)W + (1/n)B +### Mathematical Derivation + +Consider $m$ chains, each of length $n$, sampling a scalar parameter $\theta$. Let $\theta_{jt}$ denote the $t$-th sample from chain $j$. + +**Step 1: Chain means and overall mean.** Compute the mean of each chain and the grand mean across all chains: + +```math +\bar{\theta}_j = \frac{1}{n}\sum_{t=1}^{n}\theta_{jt}, \qquad \bar{\theta}_{..} = \frac{1}{m}\sum_{j=1}^{m}\bar{\theta}_j ``` +**Step 2: Between-chain variance $B$.** This measures how much the chain means differ from each other. Large $B$ relative to the within-chain variability indicates that chains have not converged to the same distribution: + +```math +B = \frac{n}{m-1}\sum_{j=1}^{m}\left(\bar{\theta}_j - \bar{\theta}_{..}\right)^2 +``` + +The factor $n/(m-1)$ scales $B$ so that it estimates the variance of $\theta$ under stationarity (the scaling by $n$ converts from variance-of-means to variance-of-individual-draws). + +**Step 3: Within-chain variance $W$.** This is the average of the individual chain variances. Each chain's variance $s_j^2$ uses the Bessel-corrected estimator: + +```math +s_j^2 = \frac{1}{n-1}\sum_{t=1}^{n}\left(\theta_{jt} - \bar{\theta}_j\right)^2 +``` + +```math +W = \frac{1}{m}\sum_{j=1}^{m}s_j^2 +``` + +**Step 4: Pooled variance estimate.** The pooled estimate combines within-chain and between-chain information: + +```math +\hat{V} = \frac{n-1}{n}\,W + \frac{1}{n}\,B +``` + +This is a weighted average that has a key property: $\hat{V}$ **overestimates** the true target variance when chains have not converged, because $B$ captures the additional spread from chains being in different regions. Meanwhile, $W$ **underestimates** the true target variance because each finite chain has only explored a portion of the full parameter space. At convergence, the between-chain contribution vanishes ($B/n \to 0$ relative to $W$), and $\hat{V} \to W$. + +**Step 5: The diagnostic ratio.** The potential scale reduction factor is: + +```math +\hat{R} = \sqrt{\frac{\hat{V}}{W}} +``` + +Since $\hat{V} \geq W$ in general, we have $\hat{R} \geq 1$. At perfect convergence $\hat{R} = 1$; values substantially above 1 indicate that the chains have not mixed and further sampling is needed. + +**Split-$\hat{R}$.** Modern practice [[4]](#4) recommends splitting each chain in half before computing $\hat{R}$, which doubles the number of chains from $m$ to $2m$. This helps detect non-stationarity *within* individual chains -- for example, a chain that drifted during the first half but settled during the second half. The ***Numerics*** implementation does not perform split-$\hat{R}$ automatically; to use this approach, split each chain manually before passing them to `GelmanRubin()`. + ### Common Causes of High R̂ 1. **Insufficient warmup** - Chains haven't reached stationarity @@ -91,8 +149,10 @@ ESS quantifies number of independent samples, accounting for autocorrelation [[2 ### Computing ESS ```cs -// For single parameter series -double[] samples = /* Extract parameter samples from chain */; +// Extract samples for parameter 0 from the first chain +double[] samples = sampler.MarkovChains[0] + .Select(ps => ps.Values[0]) + .ToArray(); double ess = MCMCDiagnostics.EffectiveSampleSize(samples); @@ -142,13 +202,121 @@ else ### ESS Formula +```math +\text{ESS} = \frac{N}{1 + 2\sum_{k=1}^{K} \rho_k} +``` +where $N$ is the number of samples, $\rho_k$ is the autocorrelation at lag $k$, and the sum is truncated when $\rho_k$ becomes negligible. + +### Mathematical Derivation + +The ESS formula arises from analyzing the variance of the sample mean of a correlated sequence. For a stationary process $\lbrace\theta_1, \theta_2, \ldots, \theta_N\rbrace$ with marginal variance $\sigma^2$ and autocorrelation function $\rho_k = \text{Corr}(\theta_t, \theta_{t+k})$, the variance of the sample mean $\bar{\theta} = \frac{1}{N}\sum_{t=1}^{N}\theta_t$ is: + +```math +\text{Var}(\bar{\theta}) = \frac{\sigma^2}{N}\left(1 + 2\sum_{k=1}^{N-1}\left(1 - \frac{k}{N}\right)\rho_k\right) +``` + +For large $N$, the $(1 - k/N)$ correction becomes negligible for the lags that matter, and the expression simplifies to: + +```math +\text{Var}(\bar{\theta}) \approx \frac{\sigma^2}{N}\left(1 + 2\sum_{k=1}^{\infty}\rho_k\right) +``` + +If the samples were independent, we would have $\rho_k = 0$ for all $k \geq 1$, giving $\text{Var}(\bar{\theta}) = \sigma^2/N$. The effective sample size is defined as the number of *independent* samples that would give the same variance for the sample mean: + +```math +\frac{\sigma^2}{\text{ESS}} = \frac{\sigma^2}{N}\left(1 + 2\sum_{k=1}^{\infty}\rho_k\right) +``` + +Solving for ESS: + +```math +\text{ESS} = \frac{N}{1 + 2\sum_{k=1}^{\infty}\rho_k} +``` + +The quantity $\tau = 1 + 2\sum_{k=1}^{\infty}\rho_k$ is called the **integrated autocorrelation time**. It represents how many MCMC iterations correspond to one independent draw: $\text{ESS} = N/\tau$. + +**Truncation strategy.** In practice, the infinite sum must be truncated. The ***Numerics*** implementation uses a simple truncation rule: the sum is cut off at the first lag $k$ where $\rho_k < 0$. This works because for a well-behaved MCMC chain, the autocorrelation function decays monotonically toward zero and oscillations below zero represent noise rather than genuine correlation. + +Geyer (1992) [[3]](#3) proposed a more robust alternative called the **initial positive sequence estimator**, which sums consecutive *pairs* of autocorrelations $(\rho_{2k} + \rho_{2k+1})$ and stops when a pair sum becomes negative. This approach is theoretically guaranteed to produce a non-negative variance estimate. The ***Numerics*** implementation uses the simpler first-negative truncation, which is adequate for chains with good mixing behavior. + +**Multi-chain ESS.** When $M$ chains of length $N$ are available, the implementation computes the autocorrelation sum $\rho_m$ for each chain $m$ separately, then averages across chains: + +```math +\bar{\rho} = \frac{1}{M}\sum_{m=1}^{M}\rho_m \qquad \text{where} \quad \rho_m = \sum_{k=1}^{K_m}\hat{\rho}_k^{(m)} ``` -ESS = N / (1 + 2·Σ ρ_k) -Where: -- N = number of samples -- ρ_k = autocorrelation at lag k -- Sum until ρ_k becomes negligible +Here $K_m$ is the truncation point for chain $m$ (the first lag at which the autocorrelation is negative). The total effective sample size is then: + +```math +\text{ESS} = \frac{N \cdot M}{1 + 2\bar{\rho}} +``` + +This is capped at $N \cdot M$ (the total number of samples) since the effective sample size cannot exceed the actual number of draws. + +### ESS Requirements + +The minimum ESS needed depends on what posterior summary you are estimating: + +| Inference Goal | Minimum ESS | Rationale | +|---------------|------------|-----------| +| Posterior mean | 100 | MCSE is approximately 10% of posterior SD | +| Posterior standard deviation | 200 | Variance estimation requires more samples than mean estimation | +| 95% credible interval | 400 | Quantile estimation demands greater precision in the tails | +| Tail probabilities (e.g., $P(\theta > c)$) | 1000+ | Extreme quantiles are estimated from sparse tail samples | + +These thresholds are guidelines, not strict rules. For life-safety applications, err on the side of larger ESS. + +### ESS per Second + +When comparing MCMC samplers, ESS alone is not sufficient -- the computational cost per iteration matters. The **ESS per second** metric accounts for this: + +```math +\text{ESS/s} = \frac{\text{ESS}}{T_{\text{compute}}} +``` + +where $T_{\text{compute}}$ is the total wall-clock time for sampling. This metric is critical for sampler selection: Hamiltonian Monte Carlo (HMC) typically achieves higher ESS per iteration than Random Walk Metropolis-Hastings (RWMH) because HMC's proposals are guided by gradient information and produce less correlated samples. However, each HMC iteration requires evaluating the gradient of the log-posterior (and often multiple leapfrog steps), making it more expensive per iteration. The optimal sampler is the one that maximizes ESS/s for the problem at hand. + +## Monte Carlo Standard Error (MCSE) + +The Monte Carlo Standard Error quantifies the precision of posterior estimates due to finite sampling [[5]](#5). While the posterior standard deviation describes uncertainty about the parameter, the MCSE describes uncertainty about the *estimate itself*. + +### Formula + +For a posterior mean estimate $\bar{\theta}$ computed from MCMC output with posterior standard deviation $\text{SD}(\theta)$ and effective sample size ESS: + +```math +\text{MCSE} = \frac{\text{SD}(\theta)}{\sqrt{\text{ESS}}} +``` + +This is the standard error of the Monte Carlo estimate of $\mathbb{E}_\pi[\theta]$. It tells you how much the posterior mean would vary if you repeated the entire MCMC run. + +### Interpretation + +A useful rule of thumb is that the MCSE should be less than 5% of the posterior standard deviation: + +```math +\frac{\text{MCSE}}{\text{SD}(\theta)} = \frac{1}{\sqrt{\text{ESS}}} < 0.05 \quad \Longrightarrow \quad \text{ESS} > 400 +``` + +This means your Monte Carlo error is small relative to the inherent posterior uncertainty. For life-safety applications, a more stringent threshold (e.g., MCSE/SD < 0.02, requiring ESS > 2500) may be appropriate. + +### Computing MCSE + +```cs +// Compute MCSE from ESS and posterior standard deviation +double[] values = samples.Select(s => s.Values[0]).ToArray(); +double sd = Statistics.StandardDeviation(values); +double mcse = sd / Math.Sqrt(ess[0]); + +Console.WriteLine($"Posterior SD: {sd:F6}"); +Console.WriteLine($"ESS: {ess[0]:F0}"); +Console.WriteLine($"MCSE: {mcse:F6}"); +Console.WriteLine($"MCSE/SD ratio: {mcse / sd:P2}"); + +if (mcse / sd > 0.05) + Console.WriteLine("⚠ MCSE is large relative to posterior SD - increase sampling"); +else + Console.WriteLine("✓ MCSE is acceptably small"); ``` ## Autocorrelation @@ -165,8 +333,8 @@ Console.WriteLine("-----|------"); for (int lag = 0; lag <= 20; lag += 5) { - // Access from avgACF[parameter][chain, lag] - double acf = avgACF[0][0, lag]; // Parameter 0, Chain 0 + // Access from avgACF[parameter][lag, column] where column 1 has ACF values + double acf = avgACF[0][lag, 1]; // Parameter 0, average ACF at this lag Console.WriteLine($"{lag,4} | {acf,5:F3}"); } @@ -185,6 +353,22 @@ for (int lag = 0; lag <= 20; lag += 5) ## Minimum Sample Size +The Raftery-Lewis method provides a theoretical lower bound on the number of MCMC samples needed to estimate a particular posterior quantile with specified precision. Given a quantile of interest $q$, tolerance $r$, and desired probability $s$, the minimum sample size is: + +```math +N_{\min} = \frac{q(1 - q)\left[z_{(1+s)/2}\right]^2}{r^2} +``` + +where $z_{(1+s)/2}$ is the standard normal quantile (inverse CDF) evaluated at $(1+s)/2$. This formula arises from the normal approximation to the binomial: the indicator $I(\theta \leq \theta_q)$ has variance $q(1-q)$ under the posterior, and $z_{(1+s)/2}$ ensures that the estimate falls within tolerance $r$ of the true quantile with probability $s$. The ***Numerics*** implementation rounds the result to the nearest hundred. + +For example, to estimate the 99th percentile ($q = 0.99$) within $\pm 0.01$ ($r = 0.01$) with 95% confidence ($s = 0.95$): + +```math +N_{\min} = \frac{0.99 \times 0.01 \times (1.96)^2}{0.01^2} = \frac{0.0099 \times 3.8416}{0.0001} \approx 380 +``` + +Note that this is a lower bound assuming independent samples. With autocorrelated MCMC output, the actual number of iterations needed is $N_{\min} \times \tau$, where $\tau$ is the integrated autocorrelation time ($\tau = N/\text{ESS}$). + Determine required sample size for desired precision: ```cs @@ -223,8 +407,8 @@ sampler.Sample(); Console.WriteLine("MCMC Convergence Diagnostics"); Console.WriteLine("=" + new string('=', 60)); -// Step 2: Extract samples -var samples = sampler.ParameterSets; +// Step 2: Extract samples (flatten all chains) +var samples = sampler.Output.SelectMany(chain => chain).ToList(); int nParams = samples[0].Values.Length; Console.WriteLine($"\nSampling Summary:"); @@ -232,11 +416,11 @@ Console.WriteLine($" Chains: {sampler.NumberOfChains}"); Console.WriteLine($" Warmup: {sampler.WarmupIterations}"); Console.WriteLine($" Iterations: {sampler.Iterations}"); Console.WriteLine($" Thinning: {sampler.ThinningInterval}"); -Console.WriteLine($" Total samples: {samples.Length}"); +Console.WriteLine($" Total samples: {samples.Count}"); // Step 3: Check Gelman-Rubin Console.WriteLine($"\nGelman-Rubin Statistics:"); -var chains = ExtractChains(sampler); // Helper function +var chains = sampler.MarkovChains.ToList(); double[] rhat = MCMCDiagnostics.GelmanRubin(chains, sampler.WarmupIterations); bool converged = true; @@ -252,9 +436,11 @@ Console.WriteLine($"\nEffective Sample Size:"); double[] ess = MCMCDiagnostics.EffectiveSampleSize(chains, out double[][,] acf); int minESS = (int)ess.Min(); +// Use MarkovChains total sample count for efficiency (ESS is computed from MarkovChains, not Output) +int chainSampleCount = chains.Sum(c => c.Count); for (int i = 0; i < nParams; i++) { - double efficiency = ess[i] / samples.Length; + double efficiency = ess[i] / chainSampleCount; Console.WriteLine($" θ{i}: ESS = {ess[i]:F0} ({efficiency:P1} efficiency)"); } @@ -287,7 +473,7 @@ Plot parameter values vs. iteration: Console.WriteLine("Export trace data for plotting:"); Console.WriteLine("Iteration | Parameter Values"); -for (int i = 0; i < Math.Min(samples.Length, 100); i++) +for (int i = 0; i < Math.Min(samples.Count, 100); i++) { Console.Write($"{i,9} | "); foreach (var val in samples[i].Values) @@ -311,10 +497,10 @@ for (int param = 0; param < nParams; param++) Console.WriteLine($"\nParameter {param} summary:"); Console.WriteLine($" Mean: {values.Average():F4}"); - Console.WriteLine($" Median: {Statistics.Percentile(values.OrderBy(v => v).ToArray(), 50):F4}"); + Console.WriteLine($" Median: {Statistics.Percentile(values, 0.50):F4}"); Console.WriteLine($" SD: {Statistics.StandardDeviation(values):F4}"); - Console.WriteLine($" 95% CI: [{Statistics.Percentile(values.OrderBy(v => v).ToArray(), 2.5):F4}, " + - $"{Statistics.Percentile(values.OrderBy(v => v).ToArray(), 97.5):F4}]"); + Console.WriteLine($" 95% CI: [{Statistics.Percentile(values, 0.025):F4}, " + + $"{Statistics.Percentile(values, 0.975):F4}]"); } ``` @@ -462,29 +648,35 @@ Console.WriteLine($" Chains: {sampler.NumberOfChains}"); Console.WriteLine($" Warmup: {sampler.WarmupIterations}"); Console.WriteLine($" Iterations: {sampler.Iterations}"); Console.WriteLine($" Thinning: {sampler.ThinningInterval}"); -Console.WriteLine($" Total samples: {samples.Length}"); +Console.WriteLine($" Total samples: {samples.Count}"); Console.WriteLine($" R̂ range: [{rhat.Min():F3}, {rhat.Max():F3}]"); Console.WriteLine($" ESS range: [{ess.Min():F0}, {ess.Max():F0}]"); ``` ### 5. Iterative Improvement +**Note:** Setting any sampler property (e.g., `Iterations`) calls `Reset()` internally, which clears all chains and output. This means increasing `Iterations` and calling `Sample()` runs a completely new simulation with the updated iteration count -- it does not extend the previous run. + ```cs int iteration = 1; +int totalIterations = sampler.Iterations; while (rhat.Max() > 1.05 || ess.Min() < 200) { - Console.WriteLine($"\nIteration {iteration}: Extending sampling..."); - - sampler.Iterations += 5000; + totalIterations += 5000; + Console.WriteLine($"\nIteration {iteration}: Restarting with {totalIterations} iterations..."); + + // Setting Iterations triggers Reset(), clearing all previous chains/output. + // Sample() then runs a fresh simulation with the new iteration count. + sampler.Iterations = totalIterations; sampler.Sample(); - + // Recompute diagnostics - chains = ExtractChains(sampler); + chains = sampler.MarkovChains.ToList(); rhat = MCMCDiagnostics.GelmanRubin(chains, sampler.WarmupIterations); ess = MCMCDiagnostics.EffectiveSampleSize(chains, out _); - + iteration++; - + if (iteration > 5) { Console.WriteLine("Convergence issues persist - check model/sampler"); @@ -510,6 +702,12 @@ while (rhat.Max() > 1.05 || ess.Min() < 200) [2] Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). *Bayesian Data Analysis* (3rd ed.). CRC Press. +[3] Geyer, C. J. (1992). Practical Markov chain Monte Carlo. *Statistical Science*, 7(4), 473-483. + +[4] Vehtari, A., Gelman, A., Simpson, D., Carpenter, B., & Bürkner, P.-C. (2021). Rank-normalization, folding, and localization: An improved R̂ for assessing convergence of MCMC. *Bayesian Analysis*, 16(2), 667-718. + +[5] Flegal, J. M., Haran, M., & Jones, G. L. (2008). Markov chain Monte Carlo: Can we trust the third significant figure? *Statistical Science*, 23(2), 250-260. + --- [← Previous: MCMC Methods](mcmc.md) | [Back to Index](../index.md) diff --git a/docs/sampling/mcmc.md b/docs/sampling/mcmc.md index df78f065..a639e4b7 100644 --- a/docs/sampling/mcmc.md +++ b/docs/sampling/mcmc.md @@ -1,6 +1,6 @@ # MCMC Sampling -[← Previous: Goodness-of-Fit](../statistics/goodness-of-fit.md) | [Back to Index](../index.md) | [Next: Convergence Diagnostics →](convergence-diagnostics.md) +[← Previous: Random Generation](random-generation.md) | [Back to Index](../index.md) | [Next: Convergence Diagnostics →](convergence-diagnostics.md) Markov Chain Monte Carlo (MCMC) methods sample from complex posterior distributions that are difficult to sample directly. The ***Numerics*** library provides multiple MCMC samplers for Bayesian inference, uncertainty quantification, and parameter estimation with full posterior distributions [[1]](#1). @@ -13,8 +13,61 @@ Markov Chain Monte Carlo (MCMC) methods sample from complex posterior distributi | **DEMCz** | Differential Evolution MCMC | High dimensions, multimodal | Population-based, efficient | | **DEMCzs** | DE-MCMC with snooker update | Very high dimensions | Enhanced DE-MCMC | | **HMC** | Hamiltonian Monte Carlo | Smooth posteriors | Uses gradient information | +| **NUTS** | No-U-Turn Sampler | General smooth posteriors | Auto-tuning HMC | | **Gibbs** | Gibbs Sampler | Conditional distributions available | No rejections | +## MCMC Fundamentals + +Before using any specific sampler, it helps to understand the core theory that underpins all MCMC methods. + +### Target Distribution + +The goal of MCMC is to draw samples from a posterior distribution that is known only up to a normalizing constant. By Bayes' theorem, the posterior is proportional to the product of the prior and the likelihood: + +```math +\pi(\theta \mid y) \propto \pi(\theta) \cdot L(y \mid \theta) +``` + +Taking the logarithm, which is how the Numerics library works internally: + +```math +\log \pi(\theta \mid y) = \log \pi(\theta) + \log L(y \mid \theta) + \text{const} +``` + +The `LogLikelihoodFunction` delegate in Numerics should return the sum $\log \pi(\theta) + \log L(y \mid \theta)$, i.e., the unnormalized log-posterior. + +### Detailed Balance + +A Markov chain with transition kernel $T(\theta \to \theta')$ satisfies **detailed balance** with respect to $\pi$ if: + +```math +\pi(\theta) \, T(\theta \to \theta') = \pi(\theta') \, T(\theta' \to \theta) +``` + +This reversibility condition guarantees that $\pi$ is a stationary distribution of the chain. All Metropolis-Hastings-based samplers in Numerics (RWMH, ARWMH, DEMCz, HMC, NUTS) enforce detailed balance through the accept/reject step, while Gibbs sampling satisfies it by construction when sampling from exact conditional distributions. + +### Ergodicity + +For the chain to converge to the target distribution regardless of its starting point, it must be **ergodic** -- meaning it is irreducible (can reach any region of positive probability) and aperiodic (does not cycle deterministically). In practice, ergodicity requires that the proposal distribution has support that covers the full posterior. The ARWMH sampler explicitly ensures ergodicity by mixing in a small identity-covariance proposal with probability $\beta$. + +### Mixing + +Mixing describes how quickly the chain "forgets" its starting point and produces effectively independent samples. Good mixing means low autocorrelation between successive states. Gradient-based samplers (HMC, NUTS) typically mix much faster than random-walk samplers because they follow the geometry of the posterior rather than exploring by diffusion. The **effective sample size (ESS)** quantifies mixing: a well-mixing chain has ESS close to the actual number of post-warmup samples. + +### Acceptance Rate Guidelines + +The acceptance rate is the fraction of proposed moves that are accepted. Acceptance rates that are too high indicate overly timid proposals (small steps), while rates that are too low mean the proposals are too aggressive. The following table summarizes established optimal acceptance rates from the literature: + +| Sampler | Optimal Acceptance Rate | Reference | +|---------|------------------------|-----------| +| RWMH ($d = 1$) | ~44% | Roberts et al. (1997) [[7]](#7) | +| RWMH ($d \to \infty$) | ~23.4% | Roberts et al. (1997) [[7]](#7) | +| HMC | ~65% | Neal (2011) [[5]](#5) | +| NUTS | ~80% (target $\delta$) | Hoffman & Gelman (2014) [[6]](#6) | +| DEMCz | Varies by dimension | ter Braak & Vrugt (2008) [[4]](#4) | + +These are rules of thumb. In practice, monitor the acceptance rates reported by `MCMCResults.AcceptanceRates` and adjust sampler settings if rates fall outside the expected range. + ## Common MCMC Interface All samplers inherit from `MCMCSampler` base class with common properties: @@ -22,7 +75,7 @@ All samplers inherit from `MCMCSampler` base class with common properties: ```cs // Configuration int PRNGSeed // Random seed (default: 12345) -int InitialIterations // Initialization phase (default: 10) +int InitialIterations // Initialization phase (base default: 10; most samplers override, e.g. RWMH/ARWMH/NUTS use 100 * NumberOfParameters) int WarmupIterations // Burn-in period (default: 1750) int Iterations // Main sampling (default: 3500) int NumberOfChains // Parallel chains (default: 4) @@ -33,10 +86,9 @@ List PriorDistributions LogLikelihood LogLikelihoodFunction // Outputs (after sampling) -ParameterSet[] ParameterSets // All samples -double[] LogLikelihoods // Log-likelihood values -double[] LogPosteriors // Log-posterior values -int[] SampleCount // Samples per chain +List[] Output // Posterior samples (one list per chain) +List[] MarkovChains // Raw MCMC chains +ParameterSet MAP // Maximum a posteriori estimate ``` ## Defining the Model @@ -107,13 +159,51 @@ This gives the log-posterior: `log(p(θ|data)) ∝ log(p(θ)) + log(p(data|θ))` ## Random Walk Metropolis-Hastings (RWMH) -The simplest and most robust MCMC algorithm [[2]](#2): +The simplest and most robust MCMC algorithm [[2]](#2). + +### Mathematical Foundation + +At each iteration, the Metropolis-Hastings algorithm proposes a new state $\theta^*$ from a proposal distribution $q(\theta^* \mid \theta)$ and accepts it with probability: + +```math +\alpha(\theta^* \mid \theta) = \min\left(1, \frac{\pi(\theta^*) \cdot q(\theta \mid \theta^*)}{\pi(\theta) \cdot q(\theta^* \mid \theta)}\right) +``` + +In the RWMH implementation (`RWMH.cs`), the proposal is a symmetric multivariate normal centered at the current state: + +```math +\theta^* \sim \mathcal{N}(\theta, \Sigma) +``` + +Because the proposal is symmetric -- $q(\theta^* \mid \theta) = q(\theta \mid \theta^*)$ -- the proposal densities cancel in the acceptance ratio, simplifying to: + +```math +\alpha = \min\left(1, \frac{\pi(\theta^*)}{\pi(\theta)}\right) +``` + +In log space, which is how the source code computes it: + +```math +\log \alpha = \log \pi(\theta^*) - \log \pi(\theta) +``` + +The proposal is accepted if $\log U \leq \log \alpha$, where $U \sim \text{Uniform}(0,1)$. If any proposed parameter falls outside its prior bounds, the proposal is immediately rejected without evaluating the log-likelihood. + +The proposal covariance matrix $\Sigma$ must be provided by the user. A common starting point is a scaled identity matrix, but better performance is achieved when $\Sigma$ approximates the shape of the posterior (e.g., from a preliminary optimization). ```cs using Numerics.Sampling.MCMC; +using Numerics.Sampling; +using Numerics.Mathematics.LinearAlgebra; +using Numerics.Data.Statistics; +using System.Linq; + +// Create proposal covariance matrix (identity scaled for initial exploration) +int nParams = priors.Count; +var proposalSigma = Matrix.Identity(nParams) * 0.1; // Create sampler -var rwmh = new RWMH(priors, logLikelihood); +var rwmh = new RWMH(priors, logLikelihood, proposalSigma); // Configure sampling rwmh.PRNGSeed = 12345; @@ -128,8 +218,8 @@ Console.WriteLine("Running RWMH sampler..."); rwmh.Sample(); // Access results -var samples = rwmh.ParameterSets; -Console.WriteLine($"Generated {samples.Length} samples"); +var samples = rwmh.Output.SelectMany(chain => chain).ToList(); +Console.WriteLine($"Generated {samples.Count} samples"); Console.WriteLine($"Samples per chain: {string.Join(", ", rwmh.SampleCount)}"); // Posterior statistics @@ -138,8 +228,8 @@ for (int i = 0; i < priors.Count; i++) var values = samples.Select(s => s.Values[i]).ToArray(); double mean = values.Average(); double std = Statistics.StandardDeviation(values); - double q025 = Statistics.Percentile(values.OrderBy(x => x).ToArray(), 2.5); - double q975 = Statistics.Percentile(values.OrderBy(x => x).ToArray(), 97.5); + double q025 = Statistics.Percentile(values, 0.025); + double q975 = Statistics.Percentile(values, 0.975); Console.WriteLine($"θ{i}: {mean:F3} ± {std:F3}, 95% CI: [{q025:F3}, {q975:F3}]"); } @@ -153,7 +243,35 @@ for (int i = 0; i < priors.Count; i++) ## Adaptive Random Walk M-H (ARWMH) -ARWMH automatically tunes the proposal distribution during warmup [[3]](#3): +ARWMH automatically tunes the proposal distribution during warmup [[3]](#3). + +### Mathematical Foundation + +The key idea behind adaptive MCMC is to learn the proposal covariance from the chain's own history, eliminating the need for manual tuning [[8]](#8). The optimal scaling theory of Roberts and Rosenthal (2001) shows that for Gaussian targets in $d$ dimensions, the optimal proposal covariance is: + +```math +\Sigma_{\text{opt}} = \frac{2.38^2}{d} \, \Sigma_{\text{target}} +``` + +where $\Sigma_{\text{target}}$ is the covariance of the target distribution. Since the target covariance is unknown, ARWMH estimates it online from the chain history. + +In the Numerics implementation (`ARWMH.cs`), the proposal at iteration $t$ uses a mixture: + +```math +\theta^* \sim \begin{cases} \mathcal{N}\!\left(\theta, \, \frac{0.1^2}{d} \, I_d\right) & \text{with probability } \beta \\ \mathcal{N}\!\left(\theta, \, \frac{2.38^2}{d} \, \hat{\Sigma}_t\right) & \text{with probability } 1-\beta \end{cases} +``` + +where: + +- $d$ is the number of parameters (`NumberOfParameters`) +- $\beta = 0.05$ by default (the `Beta` property) +- $\hat{\Sigma}_t$ is the empirical covariance matrix computed as a running covariance of accepted samples (and current states after warmup) +- $I_d$ is the $d$-dimensional identity matrix +- The scale factor $s = 2.38^2/d$ is the `Scale` property + +The small identity component (used with probability $\beta$, and also for the first $100 \times d$ samples) ensures **ergodicity**: even if the adaptive covariance estimate is poor, the chain can still reach any region of the parameter space. + +The acceptance criterion is identical to RWMH -- since both proposal components are symmetric multivariate normals centered at $\theta$, the Hastings ratio simplifies to the posterior ratio. ```cs var arwmh = new ARWMH(priors, logLikelihood); @@ -167,10 +285,10 @@ arwmh.NumberOfChains = 4; Console.WriteLine("Running Adaptive RWMH sampler..."); arwmh.Sample(); -var samples = arwmh.ParameterSets; -Console.WriteLine($"Generated {samples.Length} samples"); +var samples = arwmh.Output.SelectMany(chain => chain).ToList(); +Console.WriteLine($"Generated {samples.Count} samples"); -// ARWMH adapts proposal covariance to achieve ~23% acceptance rate +// ARWMH adapts the proposal covariance matrix during sampling (it does not explicitly target a specific acceptance rate) Console.WriteLine("ARWMH automatically tuned proposal during warmup"); ``` @@ -187,7 +305,32 @@ Console.WriteLine("ARWMH automatically tuned proposal during warmup"); ## Differential Evolution MCMC (DEMCz) -Population-based sampler using differential evolution [[4]](#4): +Population-based sampler using differential evolution [[4]](#4). + +### Mathematical Foundation + +DEMCz combines differential evolution (DE) with MCMC by using a population of chains and a history of past states to generate proposals. The mutation formula from the source code (`DEMCz.cs`) is: + +```math +\theta^*_i = \theta_i + \gamma \, (z_{R_1} - z_{R_2}) + e +``` + +where: + +- $\gamma = 2.38 / \sqrt{2d}$ is the default jump rate (`Jump` property), with $d$ the number of parameters +- $z_{R_1}$ and $z_{R_2}$ are two randomly selected states from the **population matrix** (a memory of past states from all chains) +- $e \sim \mathcal{N}(0, b^2)$ is a small noise perturbation with default $b = 10^{-3}$ (`Noise` property) +- $R_1$ and $R_2$ are drawn uniformly without replacement from $\lbrace 1, 2, \ldots, M\rbrace$, where $M$ is the current size of the population matrix + +The proposal is accepted using the standard Metropolis ratio in log space: + +```math +\log \alpha = \log \pi(\theta^*) - \log \pi(\theta) +``` + +To enable **mode-jumping** in multimodal posteriors, the jump rate $\gamma$ is set to $1.0$ with probability equal to `JumpThreshold` (default 0.1). When $\gamma = 1$, the proposal jumps the full difference between two past states, which can bridge gaps between separated modes. + +The key insight of DEMCz is that the population matrix $Z$ serves as a memory of past states from **all** chains, providing a rich set of difference vectors for generating proposals. This eliminates the need for a manually specified proposal covariance matrix -- the population automatically learns the scale and orientation of the posterior. ```cs var demcz = new DEMCz(priors, logLikelihood); @@ -201,8 +344,8 @@ demcz.Iterations = 5000; Console.WriteLine("Running DE-MCMC sampler..."); demcz.Sample(); -var samples = demcz.ParameterSets; -Console.WriteLine($"Generated {samples.Length} samples from {demcz.NumberOfChains} chains"); +var samples = demcz.Output.SelectMany(chain => chain).ToList(); +Console.WriteLine($"Generated {samples.Count} samples from {demcz.NumberOfChains} chains"); // DEMCz is particularly effective for multimodal posteriors ``` @@ -238,7 +381,48 @@ demczs.Sample(); ## Hamiltonian Monte Carlo (HMC) -Uses gradient information for efficient sampling [[5]](#5): +Uses gradient information for efficient sampling [[5]](#5) [[9]](#9). + +### Mathematical Foundation + +HMC augments the parameter space with auxiliary momentum variables $\phi$ and simulates Hamiltonian dynamics to generate distant, high-quality proposals. The Hamiltonian is defined as: + +```math +H(\theta, \phi) = U(\theta) + K(\phi) = -\log \pi(\theta) + \frac{1}{2} \phi^T M^{-1} \phi +``` + +where $U(\theta) = -\log \pi(\theta)$ is the **potential energy** (negative log-posterior) and $K(\phi)$ is the **kinetic energy**. The mass matrix $M$ is diagonal in the Numerics implementation (the `Mass` vector property). + +Hamilton's equations of motion govern the dynamics: + +```math +\frac{d\theta}{dt} = \frac{\partial H}{\partial \phi} = M^{-1}\phi, \qquad \frac{d\phi}{dt} = -\frac{\partial H}{\partial \theta} = \nabla \log \pi(\theta) +``` + +These continuous dynamics are approximated using the **leapfrog integrator**, as implemented in `HMC.cs`: + +1. Half-step momentum: $\phi \leftarrow \phi + \frac{\varepsilon}{2} \nabla \log \pi(\theta)$ +2. Full-step position: $\theta \leftarrow \theta + \varepsilon \, M^{-1} \phi$ +3. Half-step momentum: $\phi \leftarrow \phi + \frac{\varepsilon}{2} \nabla \log \pi(\theta)$ + +Steps 2-3 are repeated for $L$ leapfrog steps. After the trajectory, a Metropolis correction accounts for numerical integration error: + +```math +\alpha = \min\left(1, \exp\!\left(-H(\theta^*, \phi^*) + H(\theta, \phi)\right)\right) +``` + +In log space, the source code computes this as: + +```math +\log \alpha = \left[\log \pi(\theta^*) - \frac{1}{2}\phi^{*T} M^{-1} \phi^*\right] - \left[\log \pi(\theta) - \frac{1}{2}\phi^T M^{-1} \phi\right] +``` + +**Implementation details from `HMC.cs`:** + +- The step size $\varepsilon$ is **jittered**: each iteration draws $\varepsilon \sim \text{Uniform}(0, \, 2\varepsilon_0)$ where $\varepsilon_0$ is the `StepSize` property. This avoids resonant trajectories. +- The number of leapfrog steps $L$ is **jittered**: each iteration draws $L \sim \text{UniformDiscrete}(1, \, 2L_0)$ where $L_0$ is the `Steps` property. +- The mass vector $M$ is diagonal (default: identity). Users can set it via the `mass` constructor parameter. +- If no gradient function is provided, numerical finite differences are used via `NumericalDerivative.Gradient`, with probes clamped to prior bounds. ```cs var hmc = new HMC(priors, logLikelihood); @@ -252,8 +436,8 @@ Console.WriteLine("Running Hamiltonian Monte Carlo..."); hmc.Sample(); // HMC produces high-quality samples with lower autocorrelation -var samples = hmc.ParameterSets; -Console.WriteLine($"Generated {samples.Length} high-quality samples"); +var samples = hmc.Output.SelectMany(chain => chain).ToList(); +Console.WriteLine($"Generated {samples.Count} high-quality samples"); ``` **When to use HMC:** @@ -272,12 +456,140 @@ Console.WriteLine($"Generated {samples.Length} high-quality samples"); - Less robust to discontinuities - More complex to tune +## No-U-Turn Sampler (NUTS) + +NUTS automatically tunes the trajectory length that HMC requires as a manual setting, making it the recommended gradient-based sampler for most problems [[6]](#6). + +### Mathematical Foundation + +NUTS eliminates HMC's most sensitive tuning parameter -- the number of leapfrog steps $L$ -- by automatically determining when to stop the trajectory. It builds a balanced binary tree of leapfrog states using the following procedure: + +1. Sample momentum $\phi \sim \mathcal{N}(0, M)$ +2. Recursively double the trajectory in a randomly chosen direction (forward or backward) +3. Stop when a **U-turn** is detected or the maximum tree depth is reached + +The **U-turn criterion** (from `NUTS.cs`) checks whether the trajectory endpoints are moving apart or starting to return: + +```math +(\theta^+ - \theta^-) \cdot \phi^- < 0 \quad \text{or} \quad (\theta^+ - \theta^-) \cdot \phi^+ < 0 +``` + +where $\theta^+$ and $\theta^-$ are the forward and backward endpoints of the trajectory, and $\phi^+$ and $\phi^-$ are their corresponding momenta. When either dot product is negative, the trajectory has started to curve back on itself. + +**Candidate selection** uses multinomial sampling weighted by $\exp(-H)$: + +```math +P(\text{select } \theta_j) = \frac{\exp(-H(\theta_j, \phi_j))}{\sum_k \exp(-H(\theta_k, \phi_k))} +``` + +This is implemented via log-sum-exp arithmetic for numerical stability. + +**Step size adaptation** uses the dual averaging scheme from Hoffman and Gelman (2014), Algorithm 5. The running statistic $\bar{H}_m$ is updated at each adaptation step $m$: + +```math +\bar{H}_m = \left(1 - \frac{1}{m + t_0}\right)\bar{H}_{m-1} + \frac{1}{m + t_0}(\delta - \alpha_m) +``` + +```math +\log \varepsilon_m = \mu - \frac{\sqrt{m}}{\gamma}\bar{H}_m +``` + +where: + +- $\delta = 0.80$ is the target acceptance rate (`DELTA_TARGET`) +- $\gamma = 0.05$ is the adaptation regularization (`GAMMA`) +- $t_0 = 10$ prevents early instability (`T0`) +- $\mu = \log(10 \cdot \varepsilon_0)$ is the bias point, with $\varepsilon_0$ the initial step size +- $\alpha_m$ is the average Metropolis acceptance probability from the current tree + +The smoothed step size uses an exponential moving average with decay exponent $\kappa = 0.75$: + +```math +\log \bar{\varepsilon}_m = m^{-\kappa} \log \varepsilon_m + (1 - m^{-\kappa}) \log \bar{\varepsilon}_{m-1} +``` + +After warmup, the step size is fixed to $\exp(\log \bar{\varepsilon})$. + +**Implementation details from `NUTS.cs`:** + +- `MaxTreeDepth` defaults to 10, capping trajectories at $2^{10} = 1024$ leapfrog steps +- Divergence threshold: if $H - H_0 > 1000$, the trajectory is considered divergent and tree-building stops +- NUTS always accepts a candidate from the tree (acceptance is built into the multinomial weighting), so `AcceptCount` increments every iteration +- Step size adaptation occurs only during the warmup phase, with step sizes clamped to $[10^{-10}, \, 10^{5}]$ + +```cs +var nuts = new NUTS(priors, logLikelihood); + +// NUTS-specific settings +nuts.NumberOfChains = 4; +nuts.WarmupIterations = 1000; // Step size adapts during warmup +nuts.Iterations = 2000; + +// Optional: set step size and max tree depth +// nuts = new NUTS(priors, logLikelihood, stepSize: 0.5, maxTreeDepth: 10); + +Console.WriteLine("Running No-U-Turn Sampler..."); +nuts.Sample(); + +// Analyze results using MCMCResults +var results = new MCMCResults(nuts); + +Console.WriteLine($"Parameter 0 - Mean: {results.ParameterResults[0].SummaryStatistics.Mean:F3}"); +Console.WriteLine($"Parameter 0 - Median: {results.ParameterResults[0].SummaryStatistics.Median:F3}"); +``` + +**When to use NUTS:** +- Smooth, differentiable posteriors (same as HMC) +- When you don't want to tune the number of leapfrog steps +- Default gradient-based sampler for most applications +- Medium to high dimensions + +**Advantages over HMC:** +- No manual tuning of trajectory length +- Adapts step size automatically via dual averaging +- Eliminates wasteful U-turns in the trajectory +- Generally more efficient per computation + +**Key settings:** +- `stepSize`: Initial leapfrog step size (adapted during warmup) +- `maxTreeDepth`: Maximum binary tree depth (default 10, caps trajectory at 2^10 = 1024 steps) +- `gradientFunction`: Optional analytical gradient, provided as a constructor parameter only (not a settable property). If not provided, numerical finite differences are used. + ## Gibbs Sampler +### Mathematical Foundation + +The Gibbs sampler updates each parameter in turn by sampling from its **full conditional distribution** -- the distribution of one parameter given all the others and the data: + +```math +\theta_j^{(t+1)} \sim p\!\left(\theta_j \mid \theta_1^{(t+1)}, \ldots, \theta_{j-1}^{(t+1)}, \theta_{j+1}^{(t)}, \ldots, \theta_d^{(t)}, y\right) +``` + +The key property is that **there is no rejection step** -- every proposed sample is accepted. This makes Gibbs sampling highly efficient when the conditional distributions are available in closed form (e.g., conjugate prior-likelihood pairs). + +In the Numerics implementation (`Gibbs.cs`), the user provides a `Proposal` delegate with the signature `double[] Proposal(double[] parameters, Random prng)`. This delegate takes the current parameter vector and a pseudo-random number generator, and returns a new parameter vector sampled from the conditional distributions. The sampler always accepts the returned values (no Metropolis step). + +Default settings differ from other samplers: the Gibbs sampler uses 1 chain, no thinning (`ThinningInterval = 1`), minimal warmup (`WarmupIterations = 1`), and 100,000 iterations with an output buffer of 10,000 samples. + Samples each parameter conditionally given others: ```cs -var gibbs = new Gibbs(priors, logLikelihood); +// Define a proposal function that samples each parameter from its conditional distribution. +// The Gibbs.Proposal delegate signature is: double[] Proposal(double[] parameters, Random prng) +Gibbs.Proposal proposalFunction = (parameters, prng) => +{ + var proposed = (double[])parameters.Clone(); + for (int i = 0; i < priors.Count; i++) + { + // Sample each parameter from a small perturbation around the current value + proposed[i] = parameters[i] + new Normal(0, 0.1).InverseCDF(prng.NextDouble()); + // Clamp to prior bounds + proposed[i] = Math.Max(priors[i].Minimum, Math.Min(priors[i].Maximum, proposed[i])); + } + return proposed; +}; + +var gibbs = new Gibbs(priors, logLikelihood, proposalFunction); gibbs.NumberOfChains = 4; gibbs.WarmupIterations = 1500; @@ -304,7 +616,9 @@ Console.WriteLine("Gibbs sampler completed (no rejections)"); ```cs using Numerics.Distributions; using Numerics.Sampling.MCMC; +using Numerics.Sampling; using Numerics.Data.Statistics; +using System.Linq; // Generate synthetic data double trueIntercept = 2.0; @@ -315,7 +629,8 @@ var random = new MersenneTwister(123); int n = 20; double[] x = Enumerable.Range(1, n).Select(i => (double)i).ToArray(); double[] yTrue = x.Select(xi => trueIntercept + trueSlope * xi).ToArray(); -double[] y = yTrue.Select(yi => yi + new Normal(0, trueNoise).InverseCDF(random.NextDouble())).ToArray(); +var noiseDist = new Normal(0, trueNoise); +double[] y = yTrue.Select(yi => yi + noiseDist.InverseCDF(random.NextDouble())).ToArray(); Console.WriteLine($"Generated {n} data points"); Console.WriteLine($"True parameters: a={trueIntercept}, b={trueSlope}, σ={trueNoise}"); @@ -360,8 +675,8 @@ Console.WriteLine("\nRunning MCMC..."); sampler.Sample(); // Analyze results -var samples = sampler.ParameterSets; -Console.WriteLine($"\nPosterior Summary ({samples.Length} samples):"); +var samples = sampler.Output.SelectMany(chain => chain).ToList(); +Console.WriteLine($"\nPosterior Summary ({samples.Count} samples):"); Console.WriteLine("Parameter | True | Post Mean | Post SD | 95% Credible Interval"); Console.WriteLine("---------------------------------------------------------------"); @@ -370,11 +685,11 @@ double[] trueVals = { trueIntercept, trueSlope, trueNoise }; for (int i = 0; i < 3; i++) { - var vals = samples.Select(s => s.Values[i]).OrderBy(v => v).ToArray(); + var vals = samples.Select(s => s.Values[i]).ToArray(); double mean = vals.Average(); double std = Statistics.StandardDeviation(vals); - double lower = Statistics.Percentile(vals, 2.5); - double upper = Statistics.Percentile(vals, 97.5); + double lower = Statistics.Percentile(vals, 0.025); + double upper = Statistics.Percentile(vals, 0.975); Console.WriteLine($"{names[i],-9} | {trueVals[i],5:F2} | {mean,9:F3} | {std,7:F3} | [{lower:F3}, {upper:F3}]"); } @@ -385,89 +700,155 @@ double xNew = 15; var predictions = samples.Select(s => s.Values[0] + s.Values[1] * xNew).ToArray(); double predMean = predictions.Average(); double predSD = Statistics.StandardDeviation(predictions); -double predLower = Statistics.Percentile(predictions.OrderBy(p => p).ToArray(), 2.5); -double predUpper = Statistics.Percentile(predictions.OrderBy(p => p).ToArray(), 97.5); +double predLower = Statistics.Percentile(predictions, 0.025); +double predUpper = Statistics.Percentile(predictions, 0.975); Console.WriteLine($"E[y|x={xNew}] = {predMean:F2} ± {predSD:F2}"); Console.WriteLine($"95% Credible Interval: [{predLower:F2}, {predUpper:F2}]"); ``` -### Example 2: Distribution Parameter Estimation +### Example 2: Bayesian Distribution Fitting with Real Streamflow Data + +This example fits a Normal distribution to Tippecanoe River streamflow data using ARWMH, following the same methodology used in the library's unit tests. Results are validated against rstan with comparable MCMC settings. + +**Data source:** Rao, A. R. & Hamed, K. H. (2000). *Flood Frequency Analysis*. CRC Press, Table 5.1.1. +See also: [`example-data/tippecanoe-river-streamflow.csv`](../example-data/tippecanoe-river-streamflow.csv) ```cs -// Observed data from unknown GEV distribution -double[] annualMaxima = { 12500, 15300, 11200, 18700, 14100, 16800, 13400, 17200, 10500, 19300 }; +using Numerics.Distributions; +using Numerics.Sampling.MCMC; +using Numerics.Data.Statistics; +using System.Linq; + +// Tippecanoe River near Delphi, IN — 48 years of annual peak streamflow (cfs) +// Source: Rao & Hamed (2000), Table 5.1.1 +double[] annualPeaks = { + 6290, 2700, 13100, 16900, 14600, 9600, 7740, 8490, 8130, 12000, + 17200, 15000, 12400, 6960, 6500, 5840, 10400, 18800, 21400, 22600, + 14200, 11000, 12800, 15700, 4740, 6950, 11800, 12100, 20600, 14600, + 14600, 8900, 10600, 14200, 14100, 14100, 12500, 7530, 13400, 17600, + 13400, 19200, 16900, 15500, 14500, 21900, 10400, 7460 +}; -Console.WriteLine("Bayesian Estimation of GEV Parameters"); -Console.WriteLine("=" + new string('=', 50)); +Console.WriteLine("Bayesian Estimation of Normal Distribution Parameters"); +Console.WriteLine("Tippecanoe River near Delphi, IN (n=48)"); -// Prior distributions for GEV parameters [ξ, α, κ] +// Prior distributions for Normal(μ, σ) +// Use weakly informative priors centered on data scale var priors = new List { - new Normal(15000, 5000), // Location: N(15000, 5000) - new Uniform(100, 5000), // Scale: U(100, 5000) - new Uniform(-0.5, 0.5) // Shape: U(-0.5, 0.5) + new Uniform(0, 50000), // μ: Uniform over reasonable flow range + new Uniform(0, 50000) // σ: Uniform positive }; -// Log-likelihood +// Log-likelihood function LogLikelihood logLik = (theta) => { - double xi = theta[0]; - double alpha = theta[1]; - double kappa = theta[2]; - - // Check parameter validity - if (alpha <= 0) return double.NegativeInfinity; - - // Prior - double logPrior = priors[0].LogPDF(xi) + priors[1].LogPDF(alpha) + priors[2].LogPDF(kappa); - - // Likelihood - var gev = new GeneralizedExtremeValue(xi, alpha, kappa); - if (!gev.ParametersValid) return double.NegativeInfinity; - - double logData = annualMaxima.Sum(x => gev.LogPDF(x)); - - return logPrior + logData; + double mu = theta[0]; + double sigma = theta[1]; + if (sigma <= 0) return double.NegativeInfinity; + + var model = new Normal(mu, sigma); + return model.LogLikelihood(annualPeaks); }; -// Sample posterior +// Run ARWMH sampler var sampler = new ARWMH(priors, logLik); -sampler.WarmupIterations = 3000; -sampler.Iterations = 10000; +sampler.PRNGSeed = 12345; +sampler.WarmupIterations = 2000; +sampler.Iterations = 5000; sampler.NumberOfChains = 4; sampler.ThinningInterval = 10; -Console.WriteLine("Sampling posterior distribution..."); sampler.Sample(); -var samples = sampler.ParameterSets; -Console.WriteLine($"Generated {samples.Length} posterior samples\n"); +// Analyze posterior +var samples = sampler.Output.SelectMany(chain => chain).ToList(); +string[] paramNames = { "μ (mean)", "σ (std dev)" }; -// Parameter estimates -string[] paramNames = { "Location (ξ)", "Scale (α)", "Shape (κ)" }; -for (int i = 0; i < 3; i++) +Console.WriteLine($"\nPosterior Summary ({samples.Count} samples):"); +for (int i = 0; i < 2; i++) { - var vals = samples.Select(s => s.Values[i]).OrderBy(v => v).ToArray(); - Console.WriteLine($"{paramNames[i]}:"); - Console.WriteLine($" Mean: {vals.Average():F2}"); - Console.WriteLine($" Median: {Statistics.Percentile(vals, 50):F2}"); - Console.WriteLine($" 95% CI: [{Statistics.Percentile(vals, 2.5):F2}, {Statistics.Percentile(vals, 97.5):F2}]"); + var vals = samples.Select(s => s.Values[i]).ToArray(); + Console.WriteLine($" {paramNames[i]}:"); + Console.WriteLine($" Mean: {vals.Average():F1}"); + Console.WriteLine($" SD: {Statistics.StandardDeviation(vals):F1}"); + Console.WriteLine($" 95% CI: [{Statistics.Percentile(vals, 0.025):F1}, " + + $"{Statistics.Percentile(vals, 0.975):F1}]"); } -// Posterior predictive quantiles -Console.WriteLine("\nPosterior Predictive 100-year Flood:"); -var q100 = samples.Select(s => +// Expected results (validated against rstan): +// μ: mean ≈ 12664, sd ≈ 707 +// σ: mean ≈ 4844, sd ≈ 519 + +// MAP estimate +Console.WriteLine($"\nMAP estimate: μ={sampler.MAP.Values[0]:F1}, σ={sampler.MAP.Values[1]:F1}"); +``` + +### Post-Processing with MCMCResults + +The `MCMCResults` class provides structured post-processing of MCMC output, including summary statistics, convergence diagnostics, kernel density estimates, and histograms for each parameter: + +```cs +using Numerics.Sampling.MCMC; + +// After running the sampler, create results object +var results = new MCMCResults(sampler, alpha: 0.1); + +// Access structured parameter summaries +Console.WriteLine("Parameter Summary:"); +Console.WriteLine($"{"Param",-8} {"Mean",10} {"Median",10} {"SD",10} {"5%",10} {"95%",10} {"R-hat",8} {"ESS",8}"); + +for (int i = 0; i < results.ParameterResults.Length; i++) { - var dist = new GeneralizedExtremeValue(s.Values[0], s.Values[1], s.Values[2]); - return dist.InverseCDF(0.99); -}).OrderBy(q => q).ToArray(); + var stats = results.ParameterResults[i].SummaryStatistics; + Console.WriteLine($"{paramNames[i],-8} {stats.Mean,10:F2} {stats.Median,10:F2} " + + $"{stats.StandardDeviation,10:F2} {stats.LowerCI,10:F2} " + + $"{stats.UpperCI,10:F2} {stats.Rhat,8:F4} {stats.ESS,8:F0}"); +} + +// MAP and posterior mean estimates +Console.WriteLine($"\nMAP: [{string.Join(", ", results.MAP.Values.Select(v => v.ToString("F2")))}]"); +Console.WriteLine($"Posterior Mean: [{string.Join(", ", results.PosteriorMean.Values.Select(v => v.ToString("F2")))}]"); -Console.WriteLine($" Mean: {q100.Average():F0} cfs"); -Console.WriteLine($" Median: {Statistics.Percentile(q100, 50):F0} cfs"); -Console.WriteLine($" 95% CI: [{Statistics.Percentile(q100, 2.5):F0}, {Statistics.Percentile(q100, 97.5):F0}] cfs"); +// Access all posterior samples +Console.WriteLine($"\nTotal posterior samples: {results.Output.Count}"); + +// Access individual chain results +if (results.MarkovChains != null) +{ + for (int c = 0; c < results.MarkovChains.Length; c++) + { + Console.WriteLine($" Chain {c}: {results.MarkovChains[c].Count} samples"); + } +} ``` +**`MCMCResults` properties:** + +| Property | Type | Description | +|----------|------|-------------| +| `ParameterResults` | `ParameterResults[]` | Per-parameter statistics, KDE, and histograms | +| `MAP` | `ParameterSet` | Maximum a posteriori estimate | +| `PosteriorMean` | `ParameterSet` | Mean of posterior samples | +| `Output` | `List` | All posterior samples (aggregated across chains) | +| `MarkovChains` | `List[]` | Per-chain sample lists | +| `AcceptanceRates` | `double[]` | Acceptance rate for each chain | +| `MeanLogLikelihood` | `List` | Average log-likelihood per iteration | + +**`ParameterStatistics` properties** (via `ParameterResults[i].SummaryStatistics`): + +| Property | Type | Description | +|----------|------|-------------| +| `Mean` | `double` | Posterior mean | +| `Median` | `double` | Posterior median | +| `StandardDeviation` | `double` | Posterior standard deviation | +| `LowerCI` | `double` | Lower confidence interval (default 5th percentile) | +| `UpperCI` | `double` | Upper confidence interval (default 95th percentile) | +| `Rhat` | `double` | Gelman-Rubin convergence diagnostic | +| `ESS` | `double` | Effective sample size | +| `N` | `int` | Total sample count | + ## Thinning Thinning reduces autocorrelation by keeping only every nth sample: @@ -530,8 +911,35 @@ sampler.Iterations = 5000; // Kept samples - **ARWMH**: 2000-3000 (adapts during warmup) - **DEMCz**: 1500-3000 (converges faster) - **HMC**: 1000-2000 (efficient exploration) +- **NUTS**: 1000-2000 (step size adapts during warmup) - **Rule of thumb**: Warmup ≥ 50% of main iterations +## Algorithm Selection Guide + +Use the following decision tree to choose the most appropriate sampler for your problem: + +``` +Do you have gradient information (or a smooth, differentiable log-posterior)? +├── YES: Use NUTS (auto-tunes trajectory length) +│ └── If NUTS is too slow per iteration: Try HMC with manually tuned ε, L +├── NO: How many parameters? +│ ├── d ≤ 5: ARWMH (simple, robust, self-tuning) +│ ├── 5 < d ≤ 20: ARWMH or DEMCz +│ │ └── Multimodal posterior? → DEMCz (population-based exploration) +│ └── d > 20: DEMCz or DEMCzs +│ └── Very high dimensions → DEMCzs (snooker update for better mixing) +└── Conjugate model with known conditional distributions? + └── YES: Gibbs (no rejections, maximum efficiency) +``` + +**Key trade-offs:** + +- **RWMH** is the simplest algorithm and a useful baseline, but requires manual proposal tuning. Use it when you want full control or need a reference sampler. +- **ARWMH** is the recommended default for most problems without gradients. It eliminates manual tuning and adapts to posterior correlations automatically. +- **DEMCz / DEMCzs** excel at high-dimensional and multimodal problems. They require more chains (minimum 3, typically 10+) but need no proposal covariance specification. +- **HMC / NUTS** provide the best mixing per sample for smooth posteriors, but require gradient computation (analytical or numerical). NUTS is preferred over HMC because it eliminates manual trajectory-length tuning. +- **Gibbs** is maximally efficient when exact conditional distributions are available, but its applicability is limited to conjugate or conditionally tractable models. + ## Best Practices ### 1. Always Check Convergence @@ -564,7 +972,7 @@ sampler.Iterations = 5000; // Low dimensions (< 5): RWMH or ARWMH // Medium (5-20): ARWMH (default choice) // High (20+): DEMCz or DEMCzs -// Smooth posteriors: HMC +// Smooth posteriors: NUTS (preferred) or HMC // Conjugate models: Gibbs ``` @@ -593,9 +1001,16 @@ if (double.IsNaN(logLik) || double.IsInfinity(logLik)) ## Comparing Samplers ```cs +using Numerics.Sampling.MCMC; +using Numerics.Mathematics.LinearAlgebra; +using System.Linq; + +// RWMH requires a proposal covariance matrix +var rwmhProposal = Matrix.Identity(priors.Count) * 0.1; + var samplers = new (string Name, MCMCSampler Sampler)[] { - ("RWMH", new RWMH(priors, logLik)), + ("RWMH", new RWMH(priors, logLik, rwmhProposal)), ("ARWMH", new ARWMH(priors, logLik)), ("DEMCz", new DEMCz(priors, logLik)) }; @@ -604,15 +1019,66 @@ foreach (var (name, sampler) in samplers) { sampler.WarmupIterations = 2000; sampler.Iterations = 5000; - + var watch = System.Diagnostics.Stopwatch.StartNew(); sampler.Sample(); watch.Stop(); - - Console.WriteLine($"{name}: {sampler.ParameterSets.Length} samples in {watch.ElapsedMilliseconds}ms"); + + int totalSamples = sampler.Output.Sum(chain => chain.Count); + Console.WriteLine($"{name}: {totalSamples} samples in {watch.ElapsedMilliseconds}ms"); } ``` +## Self-Normalizing Importance Sampling (SNIS) + +The `SNIS` sampler performs self-normalizing importance sampling rather than iterative MCMC. It draws independent samples from a proposal distribution and re-weights them by the likelihood. This is useful when the posterior is close to the prior or when a good proposal distribution is available. + +```cs +using Numerics.Sampling.MCMC; +using Numerics.Distributions; + +// Observed data +double[] observations = { 3.2, 4.8, 5.1, 6.3, 4.9, 5.5, 6.1, 3.8, 5.0, 4.7 }; + +// Define priors +var priors = new List +{ + new Normal(0, 10), + new Uniform(0, 50) +}; + +// Log-likelihood function +// Note: use a lowercase name to avoid shadowing the LogLikelihood delegate type. +LogLikelihood computeLogLikelihood = (double[] theta) => +{ + double mu = theta[0], sigma = theta[1]; + if (sigma <= 0) return double.NegativeInfinity; + var model = new Normal(mu, sigma); + return model.LogLikelihood(observations); +}; + +// Naive Monte Carlo (sample from priors) +var snis = new SNIS(priors, computeLogLikelihood); +snis.Iterations = 100000; +snis.Sample(); + +// With a Multivariate Normal proposal distribution (importance sampling) +var proposal = new MultivariateNormal( + new double[] { 5, 10 }, // mean vector + new double[,] { { 4, 0 }, { 0, 25 } } // covariance matrix +); +var snisIS = new SNIS(priors, computeLogLikelihood, proposal); +snisIS.Iterations = 100000; +snisIS.Sample(); +``` + +**Key differences from MCMC samplers:** +- Single chain only (`NumberOfChains` = 1) +- No warmup or thinning — all samples contribute +- Produces weighted samples (weights stored in `ParameterSet.Weight`) +- No autocorrelation — samples are independent +- Best for low-dimensional problems with informative proposals + --- ## References @@ -627,6 +1093,14 @@ foreach (var (name, sampler) in samplers) [5] Neal, R. M. (2011). MCMC using Hamiltonian dynamics. *Handbook of Markov Chain Monte Carlo*, 2(11), 2. +[6] Hoffman, M. D., & Gelman, A. (2014). The No-U-Turn Sampler: Adaptively setting path lengths in Hamiltonian Monte Carlo. *Journal of Machine Learning Research*, 15(47), 1593-1623. + +[7] Roberts, G. O., Gelman, A., & Gilks, W. R. (1997). Weak convergence and optimal scaling of random walk Metropolis algorithms. *Annals of Applied Probability*, 7(1), 110-120. + +[8] Roberts, G. O., & Rosenthal, J. S. (2001). Optimal scaling for various Metropolis-Hastings algorithms. *Statistical Science*, 16(4), 351-367. + +[9] Betancourt, M. (2017). A conceptual introduction to Hamiltonian Monte Carlo. *arXiv preprint arXiv:1701.02434*. + --- -[← Previous: Goodness-of-Fit](../statistics/goodness-of-fit.md) | [Back to Index](../index.md) | [Next: Convergence Diagnostics →](convergence-diagnostics.md) +[← Previous: Random Generation](random-generation.md) | [Back to Index](../index.md) | [Next: Convergence Diagnostics →](convergence-diagnostics.md) diff --git a/docs/sampling/random-generation.md b/docs/sampling/random-generation.md index d9d02b4f..6e692a71 100644 --- a/docs/sampling/random-generation.md +++ b/docs/sampling/random-generation.md @@ -1,6 +1,6 @@ # Random Number Generation -[← Previous: Time Series](../data/time-series.md) | [Back to Index](../index.md) +[← Previous: Machine Learning](../machine-learning/machine-learning.md) | [Back to Index](../index.md) | [Next: MCMC Methods →](mcmc.md) The ***Numerics*** library provides multiple random number generation methods for different applications. These include high-quality pseudo-random generators, quasi-random sequences, and advanced sampling techniques. @@ -36,16 +36,32 @@ Console.WriteLine($"Double [0,1): {u1:F6}"); Console.WriteLine($"Double (0,1): {u4:F6}"); ``` +### How It Works + +The Mersenne Twister (MT19937) maintains an internal state vector of $N = 624$ 32-bit integers. The state evolves through a linear recurrence: + +```math +\mathbf{x}_{k+N} = \mathbf{x}_{k+M} \oplus \left(\mathbf{x}_k^{\text{upper}} \mid \mathbf{x}_{k+1}^{\text{lower}}\right) \cdot A +``` + +where $M = 397$, $\oplus$ is bitwise XOR, and $A$ is a carefully chosen matrix that ensures the period equals $2^{19937} - 1$ — a Mersenne prime, hence the name. The state holds $624 \times 32 = 19{,}968$ bits, so the generator cycles through $2^{19937} - 1$ values before repeating (a number with 6,002 digits). + +When a random number is requested, the raw state word is **tempered** through a sequence of XOR and shift operations to improve the output's equidistribution properties. The generator is 623-dimensionally equidistributed, meaning that consecutive 623-tuples of 32-bit outputs are uniformly distributed in $[0, 2^{32})^{623}$. + +The `GenRandRes53()` method combines two 32-bit outputs to produce a double with full 53-bit mantissa resolution, giving the finest granularity possible in $[0, 1)$. + **Properties:** -- Period: 2^19937 - 1 (extremely long) -- Excellent uniformity and independence +- Period: $2^{19937} - 1$ (a 6,002-digit number) +- 623-dimensionally equidistributed at 32-bit precision - Fast generation - Reproducible with seeds ### Using with Distributions ```cs +using Numerics.Sampling; using Numerics.Distributions; +using Numerics.Data.Statistics; var rng = new MersenneTwister(12345); @@ -88,14 +104,34 @@ double[] pointAt100 = sobol.SkipTo(100); Console.WriteLine($"\nPoint at index 100: ({pointAt100[0]:F4}, {pointAt100[1]:F4})"); ``` +### How It Works + +A Sobol sequence is constructed using **direction numbers** — carefully chosen integers that control how each dimension fills the unit interval. The implementation uses the Joe-Kuo direction numbers, supporting up to 21,201 dimensions with 52-bit precision. + +Each point in the sequence is generated using **Gray code** indexing. To advance from index $n$ to $n+1$, only a single XOR operation is needed per dimension: + +```math +x_i^{(n+1)} = x_i^{(n)} \oplus v_{i,c} +``` + +where $c$ is the position of the rightmost zero bit in $n$, and $v_{i,c}$ is the $c$-th direction number for dimension $i$. This makes generation $O(1)$ per point. The `SkipTo(index)` method uses Gray code conversion $G(n) = n \oplus (n \gg 1)$ to jump directly to any position in $O(\log n)$ operations. + +The key property is **low discrepancy**: the star discrepancy $D_N^*$ of $N$ Sobol points in $d$ dimensions satisfies: + +```math +D_N^* = O\left(\frac{(\log N)^d}{N}\right) +``` + +By the Koksma-Hlawka inequality, the integration error using quasi-Monte Carlo is bounded by $D_N^* \cdot V(f)$ where $V(f)$ is the variation of the integrand. Compare this to Monte Carlo's $O(1/\sqrt{N})$ rate — for smooth functions, Sobol sequences converge much faster. + **Properties:** -- Low discrepancy (better coverage than pseudo-random) -- Deterministic sequence -- Excellent for integration and optimization -- Converges faster than Monte Carlo +- Low discrepancy — points fill space more evenly than pseudo-random +- Deterministic sequence (same points every time) +- Up to 21,201 dimensions (Joe-Kuo direction numbers) +- Integration error $O((\log N)^d / N)$ vs. Monte Carlo's $O(1/\sqrt{N})$ **When to use:** -- Numerical integration (better than Monte Carlo) +- Numerical integration (often 10–100× fewer points than Monte Carlo for the same accuracy) - Parameter space exploration - Optimization initialization - Sensitivity analysis @@ -103,6 +139,8 @@ Console.WriteLine($"\nPoint at index 100: ({pointAt100[0]:F4}, {pointAt100[1]:F4 ### Sobol vs. Pseudo-Random ```cs +using Numerics.Sampling; + int n = 100; var random = new MersenneTwister(123); var sobol = new SobolSequence(2); @@ -133,6 +171,7 @@ Stratified sampling for better parameter space coverage [[3]](#3): ```cs using Numerics.Sampling; +using Numerics.Distributions; // Generate Latin Hypercube sample int sampleSize = 50; @@ -175,14 +214,34 @@ for (int i = 0; i < Math.Min(5, sampleSize); i++) } ``` +### How It Works + +Latin Hypercube Sampling divides each dimension's range $[0, 1)$ into $n$ equal strata of width $1/n$, then places exactly one sample in each stratum. For $d$ dimensions with $n$ samples, the $i$-th sample in dimension $j$ is: + +```math +x_{ij} = \frac{\pi_j(i) + U_{ij}}{n}, \quad i = 0, \ldots, n-1 +``` + +where $\pi_j$ is a random permutation of $\lbrace 0, 1, \ldots, n-1\rbrace$ (independent for each dimension) and $U_{ij} \sim \text{Uniform}(0, 1)$. The library's `Median` variant replaces $U_{ij}$ with $0.5$, placing each point at the stratum center. + +The random permutations are generated using the **Fisher-Yates shuffle**, and each dimension uses an independent Mersenne Twister seeded from a master RNG. + +The key guarantee is **marginal stratification**: when projected onto any single axis, the $n$ samples fall exactly one per stratum. This eliminates the clustering and gaps that plague simple random sampling. For estimating $E[f(X)]$, the variance of the LHS estimator satisfies: + +```math +\text{Var}[\hat{\mu}_{\text{LHS}}] \leq \text{Var}[\hat{\mu}_{\text{MC}}] +``` + +with equality only when $f$ is constant within each stratum. In practice, LHS often achieves the same accuracy as simple Monte Carlo with 5–10× fewer samples. + **Properties:** -- Stratified sampling (one sample per stratum) -- Better coverage than simple random sampling -- Reduced variance in estimates -- Efficient for small sample sizes +- Stratified sampling (exactly one sample per stratum in each dimension) +- Variance reduction over simple random sampling +- Two variants: random (within-stratum jitter) and median (stratum centers) +- Most efficient for small to medium sample sizes **When to use:** -- Monte Carlo simulation with limited budget +- Monte Carlo simulation with limited computational budget - Sensitivity analysis - Calibration with expensive models - Risk assessment studies @@ -192,6 +251,8 @@ for (int i = 0; i < Math.Min(5, sampleSize); i++) ### Example 1: Monte Carlo Integration ```cs +using Numerics.Sampling; + // Integrate f(x) = x² from 0 to 1 using different methods Func f = x => x * x; @@ -227,6 +288,10 @@ Console.WriteLine("\nQMC typically has smaller error for same sample size"); ### Example 2: Uncertainty Propagation ```cs +using Numerics.Sampling; +using Numerics.Distributions; +using Numerics.Data.Statistics; + // Model: y = a*x + b*x² where a, b are uncertain var normal_a = new Normal(2.0, 0.3); @@ -265,6 +330,8 @@ Console.WriteLine("\nLHS typically more stable with fewer samples"); ### Example 3: Global Optimization Initialization ```cs +using Numerics.Sampling; + // Initialize population for global optimization int popSize = 20; @@ -302,6 +369,9 @@ Console.WriteLine("\nLHS ensures good coverage of parameter space"); ### Example 4: Sensitivity Analysis ```cs +using Numerics.Sampling; +using Numerics.Data.Statistics; + // Compute Sobol sensitivity indices using quasi-random sampling Func model = x => @@ -385,6 +455,8 @@ for (int j = 0; j < dim; j++) ### Setting Seeds ```cs +using Numerics.Sampling; + // Pseudo-random - use same seed for reproducibility var rng1 = new MersenneTwister(12345); var rng2 = new MersenneTwister(12345); @@ -437,4 +509,4 @@ for (int i = 0; i < 3; i++) --- -[← Previous: Time Series](../data/time-series.md) | [Back to Index](../index.md) +[← Previous: Machine Learning](../machine-learning/machine-learning.md) | [Back to Index](../index.md) | [Next: MCMC Methods →](mcmc.md) diff --git a/docs/statistics/descriptive.md b/docs/statistics/descriptive.md index 81cbe59b..1031fd77 100644 --- a/docs/statistics/descriptive.md +++ b/docs/statistics/descriptive.md @@ -1,9 +1,77 @@ # Descriptive Statistics -[← Back to Index](../index.md) | [Next: Goodness-of-Fit →](goodness-of-fit.md) +[← Previous: Time Series](../data/time-series.md) | [Back to Index](../index.md) | [Next: Goodness-of-Fit →](goodness-of-fit.md) The ***Numerics*** library provides comprehensive functions for computing descriptive statistics from data samples. The `Statistics` class contains static methods for all common statistical measures, sample moments, percentiles, and specialized analyses. +## Mathematical Foundations + +This section defines the mathematical formulas underlying the descriptive statistics computed by the library. All formulas have been verified against the source implementation. The library uses numerically stable algorithms internally, but the results are mathematically equivalent to the definitions below. + +### Sample Mean + +The arithmetic mean of $n$ observations: + +```math +\bar{x} = \frac{1}{n}\sum_{i=1}^{n} x_i +``` + +### Sample Variance (Bessel's Correction) + +The unbiased estimator of the population variance divides by $n-1$ rather than $n$. This is known as Bessel's correction, and it compensates for the bias that arises when using the sample mean in place of the true population mean. The `Variance` method returns this estimator: + +```math +s^2 = \frac{1}{n-1}\sum_{i=1}^{n}(x_i - \bar{x})^2 +``` + +The source code implements this using a numerically stable one-pass recurrence rather than the naive two-pass formula shown above. The two approaches are mathematically equivalent, but the one-pass algorithm avoids catastrophic cancellation when the mean is large relative to the variance. + +### Population Variance + +When the data represents the entire population (not a sample), the `PopulationVariance` method divides by $n$: + +```math +\sigma^2 = \frac{1}{n}\sum_{i=1}^{n}(x_i - \bar{x})^2 +``` + +### Skewness (Fisher's Adjusted) + +The `Skewness` method computes the bias-corrected Fisher skewness (type 2), which adjusts the raw sample skewness for sample size: + +```math +G_1 = \frac{\sqrt{n(n-1)}}{n-2} \cdot g_1 +``` + +where $g_1 = m_3 / m_2^{3/2}$ is the sample skewness computed from the central moments $m_k = \frac{1}{n}\sum_{i=1}^{n}(x_i - \bar{x})^k$. + +**Interpretation:** +- $G_1 > 0$: right-skewed (long right tail) +- $G_1 < 0$: left-skewed (long left tail) +- $|G_1| < 0.5$: approximately symmetric + +### Excess Kurtosis (Fisher's Adjusted) + +The `Kurtosis` method computes bias-corrected excess kurtosis (type 2). Excess kurtosis subtracts 3 from Pearson's kurtosis so that the normal distribution has a value of zero: + +```math +G_2 = \frac{n(n+1)}{(n-1)(n-2)(n-3)} \cdot \frac{\sum(x_i - \bar{x})^4}{s^4} - \frac{3(n-1)^2}{(n-2)(n-3)} +``` + +where $s^2 = \frac{1}{n-1}\sum(x_i - \bar{x})^2$ is the sample variance. + +**Interpretation:** +- $G_2 > 0$: leptokurtic (heavier tails than normal) +- $G_2 < 0$: platykurtic (lighter tails than normal) +- $G_2 \approx 0$: mesokurtic (similar to normal distribution) + +### Coefficient of Variation + +The coefficient of variation expresses the standard deviation as a fraction of the mean, providing a dimensionless measure of relative variability: + +```math +\text{CV} = \frac{s}{\bar{x}} +``` + ## Basic Statistics ### Central Tendency @@ -160,7 +228,35 @@ Console.WriteLine($" Kurtosis (γ₂): {moments[3]:F3}"); ### Linear Moments (L-Moments) -Linear moments are robust alternatives to product moments [[1]](#1): +Linear moments (L-moments) are robust alternatives to conventional product moments, introduced by Hosking [[1]](#1). They are defined through probability weighted moments (PWMs), which use order statistics rather than powers of deviations from the mean. + +The PWMs are computed as: + +```math +\beta_r = \frac{1}{n}\sum_{i=1}^{n}\frac{\binom{i-1}{r}}{\binom{n-1}{r}} x_{i:n} +``` + +where $x_{i:n}$ are the order statistics (sorted data values). The library computes PWMs $\beta_0$ through $\beta_3$, from which the first four L-moments are derived: + +```math +\lambda_1 = \beta_0, \quad \lambda_2 = 2\beta_1 - \beta_0 +``` + +The `LinearMoments` method returns $\lambda_1$ (L-location), $\lambda_2$ (L-scale), and the L-moment ratios $\tau_3$ (L-skewness) and $\tau_4$ (L-kurtosis): + +```math +\tau_3 = \frac{\lambda_3}{\lambda_2}, \quad \tau_4 = \frac{\lambda_4}{\lambda_2} +``` + +where $\lambda_3$ and $\lambda_4$ are the third and fourth L-moments, computed from PWMs $\beta_0$ through $\beta_3$. + +**Why use L-moments over product moments?** + +- **Robust to outliers**: L-moments are based on order statistics, not powers of deviations, making them far less sensitive to extreme values. +- **Better estimators for small samples**: When $n < 50$, L-moment estimators have lower bias and variance than product moment estimators. +- **Unique characterization**: L-moments uniquely characterize a distribution, similar to moment generating functions. +- **Bounded L-moment ratios**: $-1 \leq \tau_3 \leq 1$ and $\tau_4 \geq (5\tau_3^2 - 1)/4$, which makes them easy to interpret and compare. +- **Standard in hydrology**: L-moments are the preferred approach for flood frequency analysis per Hosking (1990) [[1]](#1). ```cs double[] data = { 10.5, 12.3, 11.8, 15.2, 13.7, 14.1, 16.8, 12.9, 11.2, 14.5 }; @@ -182,22 +278,30 @@ Console.WriteLine($" τ₄ (L-kurtosis): {lMoments[3]:F4}"); ## Percentiles and Quantiles +The library computes percentiles using Type 7 linear interpolation, which is the default method in R and Excel. For a given probability $p \in [0, 1]$: + +```math +Q(p) = x_{\lfloor h \rfloor} + (h - \lfloor h \rfloor)(x_{\lceil h \rceil} - x_{\lfloor h \rfloor}) +``` + +where $h = (n-1)p$ and $x_i$ are the sorted data values (0-indexed). When $h$ falls exactly on an integer, the result is the corresponding order statistic; otherwise, it linearly interpolates between the two adjacent order statistics. + ### Computing Percentiles ```cs double[] data = { 10.5, 12.3, 11.8, 15.2, 13.7, 14.1, 16.8, 12.9, 11.2, 14.5 }; -// Single percentile (k as decimal: 0-100) -double median = Statistics.Percentile(data, 50); // 50th percentile -double p90 = Statistics.Percentile(data, 90); // 90th percentile -double p95 = Statistics.Percentile(data, 95); // 95th percentile +// Single percentile (k in [0, 1]) +double median = Statistics.Percentile(data, 0.50); // 50th percentile +double p90 = Statistics.Percentile(data, 0.90); // 90th percentile +double p95 = Statistics.Percentile(data, 0.95); // 95th percentile Console.WriteLine($"Median (50th percentile): {median:F2}"); Console.WriteLine($"90th percentile: {p90:F2}"); Console.WriteLine($"95th percentile: {p95:F2}"); // Multiple percentiles at once (more efficient) -double[] percentiles = Statistics.Percentile(data, new double[] { 25, 50, 75, 90, 95 }); +double[] percentiles = Statistics.Percentile(data, new double[] { 0.25, 0.50, 0.75, 0.90, 0.95 }); Console.WriteLine("\nPercentiles:"); Console.WriteLine($" 25th: {percentiles[0]:F2}"); @@ -208,7 +312,7 @@ Console.WriteLine($" 95th: {percentiles[4]:F2}"); // Note: Can specify if data is already sorted for efficiency bool isSorted = false; -double q25 = Statistics.Percentile(data, 25, dataIsSorted: isSorted); +double q25 = Statistics.Percentile(data, 0.25, dataIsSorted: isSorted); ``` ### Five-Number Summary @@ -246,11 +350,11 @@ double[] sevenNum = Statistics.SevenNumberSummary(data); Console.WriteLine("Seven-Number Summary:"); Console.WriteLine($" Minimum: {sevenNum[0]:F2}"); -Console.WriteLine($" 10th percentile: {sevenNum[1]:F2}"); +Console.WriteLine($" 5th percentile: {sevenNum[1]:F2}"); Console.WriteLine($" Q1 (25th): {sevenNum[2]:F2}"); Console.WriteLine($" Median (50th): {sevenNum[3]:F2}"); Console.WriteLine($" Q3 (75th): {sevenNum[4]:F2}"); -Console.WriteLine($" 90th percentile: {sevenNum[5]:F2}"); +Console.WriteLine($" 95th percentile: {sevenNum[5]:F2}"); Console.WriteLine($" Maximum: {sevenNum[6]:F2}"); ``` @@ -284,7 +388,42 @@ else ### Correlation -For correlation coefficients, use the `Correlation` class: +For correlation coefficients, use the `Correlation` class. Three measures are provided, each capturing a different aspect of association between two variables. + +**Pearson correlation** measures the strength and direction of the linear relationship between two variables [[3]](#3): + +```math +r = \frac{\sum_{i=1}^{n}(x_i - \bar{x})(y_i - \bar{y})}{\sqrt{\sum_{i=1}^{n}(x_i - \bar{x})^2 \cdot \sum_{i=1}^{n}(y_i - \bar{y})^2}} +``` + +**Spearman rank correlation** is the Pearson correlation applied to the ranks of the data rather than the data values themselves. This makes it a nonparametric measure of monotonic association: + +```math +\rho_s = r(\text{rank}(x), \text{rank}(y)) +``` + +**Kendall's tau-b** [[4]](#4) measures the strength of ordinal association, adjusted for ties. The library implements tau-b: + +```math +\tau_b = \frac{n_c - n_d}{\sqrt{n_1 \cdot n_2}} +``` + +where $n_c$ is the number of concordant pairs, $n_d$ is the number of discordant pairs, $n_1$ is the number of pairs not tied in $x$, and $n_2$ is the number of pairs not tied in $y$. + +**When to use which correlation:** + +| Correlation | Measures | Robust to Outliers | Handles Nonlinear | +|------------|----------|-------------------|------------------| +| Pearson | Linear association | No | No | +| Spearman | Monotonic association | Yes | Yes (monotonic) | +| Kendall's $\tau_b$ | Concordance | Yes | Yes (monotonic) | + +Rules of thumb: +- **Pearson**: use for normally distributed data with linear relationships. +- **Spearman**: use when data has outliers or the relationship is monotonic but not linear. +- **Kendall's $\tau_b$**: more robust than Spearman for small samples and has better statistical properties. Preferred when there are many tied values. + +The `Correlation` class also provides matrix versions (`Pearson(double[,])` and `Spearman(double[,])`) that compute the full $p \times p$ correlation matrix for multivariate data. ```cs using Numerics.Data.Statistics; @@ -299,7 +438,7 @@ double pearson = Correlation.Pearson(x, y); double spearman = Correlation.Spearman(x, y); // Kendall tau correlation -double kendall = Correlation.Kendall(x, y); +double kendall = Correlation.KendallsTau(x, y); Console.WriteLine($"Pearson r: {pearson:F3}"); Console.WriteLine($"Spearman ρ: {spearman:F3}"); @@ -319,6 +458,8 @@ else ### Rank Statistics ```cs +using System.Linq; + double[] data = { 5.2, 3.1, 7.8, 3.1, 9.2, 5.2 }; // Compute ranks (in-place, modifies array) @@ -348,11 +489,8 @@ using Numerics.Distributions; double[] sample = new Normal(0, 1).GenerateRandomValues(1000); // Estimate entropy using kernel density -Func pdf = x => -{ - var kde = new KernelDensity(sample, bandwidth: 0.5); - return kde.PDF(x); -}; +var kde = new KernelDensity(sample, KernelDensity.KernelType.Gaussian, 0.5); +Func pdf = x => kde.PDF(x); double entropy = Statistics.Entropy(sample, pdf); @@ -364,13 +502,35 @@ Console.WriteLine($"In bits: {entropy / Math.Log(2):F3}"); ## Jackknife Resampling -Leave-one-out resampling for standard error estimation: +The jackknife is a resampling technique that estimates the variability of a statistic $\hat{\theta}$ by systematically leaving out one observation at a time. It is particularly useful for estimating standard errors of statistics that lack closed-form variance expressions (e.g., the median, L-moment ratios, or custom estimators). + +### Jackknife Standard Error + +The `JackKnifeStandardError` method computes: + +```math +\text{SE}_{\text{jack}} = \sqrt{\frac{n-1}{n}\sum_{i=1}^{n}(\hat{\theta}_{(-i)} - \hat{\theta})^2} +``` + +where $\hat{\theta}_{(-i)}$ is the statistic computed without the $i$-th observation, and $\hat{\theta}$ is the statistic computed on the full sample. + +### Jackknife Bias Estimate + +The `JackKnifeSample` method returns the array of leave-one-out estimates $\hat{\theta}_{(-i)}$, from which the jackknife bias estimate can be computed: + +```math +\text{bias}_{\text{jack}} = (n-1)(\bar{\theta}_{(\cdot)} - \hat{\theta}) +``` + +where $\bar{\theta}_{(\cdot)} = \frac{1}{n}\sum_{i=1}^{n}\hat{\theta}_{(-i)}$ is the mean of the jackknife replications. + +Note that the library parallelizes the jackknife loop internally using `Parallel.For`, so it scales well with multi-core processors. ```cs double[] data = { 10.5, 12.3, 11.8, 15.2, 13.7, 14.1, 16.8, 12.9, 11.2, 14.5 }; // Define a statistic function (e.g., median) -Func, double> medianFunc = sample => Statistics.Percentile(sample.ToArray(), 50); +Func, double> medianFunc = sample => Statistics.Percentile(sample.ToArray(), 0.50); // Jackknife standard error double jackknifeSE = Statistics.JackKnifeStandardError(data, medianFunc); @@ -378,49 +538,71 @@ double jackknifeSE = Statistics.JackKnifeStandardError(data, medianFunc); Console.WriteLine($"Median: {medianFunc(data):F2}"); Console.WriteLine($"Jackknife SE: {jackknifeSE:F3}"); -// Get all jackknife samples -double[] jackknifeValues = Statistics.JackKnifeSample(data, medianFunc); +// Get all jackknife samples (returns null if data is empty) +double[]? jackknifeValues = Statistics.JackKnifeSample(data, medianFunc); -Console.WriteLine($"Jackknife samples: {jackknifeValues.Length}"); -Console.WriteLine($"Mean of jackknife estimates: {jackknifeValues.Average():F2}"); +if (jackknifeValues != null) +{ + Console.WriteLine($"Jackknife samples: {jackknifeValues.Length}"); + Console.WriteLine($"Mean of jackknife estimates: {jackknifeValues.Average():F2}"); +} ``` ## Practical Examples -### Example 1: Complete Data Summary +### Example 1: Complete Streamflow Data Summary + +This example analyzes the Tippecanoe River annual peak streamflow data. Statistics are validated against R's `base`, `psych`, `EnvStats`, and `lmom` packages. + +**Data source:** Rao, A. R. & Hamed, K. H. (2000). *Flood Frequency Analysis*. CRC Press, Table 5.1.1. +See also: [`example-data/tippecanoe-river-streamflow.csv`](../example-data/tippecanoe-river-streamflow.csv) ```cs using Numerics.Data.Statistics; -double[] annualRainfall = { 850, 920, 780, 1050, 890, 950, 820, 1100, 870, 980 }; +// Tippecanoe River near Delphi, IN — 48 years of annual peak streamflow (cfs) +double[] annualPeaks = { + 6290, 2700, 13100, 16900, 14600, 9600, 7740, 8490, 8130, 12000, + 17200, 15000, 12400, 6960, 6500, 5840, 10400, 18800, 21400, 22600, + 14200, 11000, 12800, 15700, 4740, 6950, 11800, 12100, 20600, 14600, + 14600, 8900, 10600, 14200, 14100, 14100, 12500, 7530, 13400, 17600, + 13400, 19200, 16900, 15500, 14500, 21900, 10400, 7460 +}; -Console.WriteLine("Annual Rainfall Analysis (mm)"); -Console.WriteLine("=" + new string('=', 50)); +Console.WriteLine("Tippecanoe River Annual Peak Streamflow Analysis (cfs)"); +Console.WriteLine("=" + new string('=', 55)); // Central tendency Console.WriteLine("\nCentral Tendency:"); -Console.WriteLine($" Mean: {Statistics.Mean(annualRainfall):F1} mm"); -Console.WriteLine($" Median: {Statistics.Percentile(annualRainfall, 50):F1} mm"); +Console.WriteLine($" Mean: {Statistics.Mean(annualPeaks):F1} cfs"); +Console.WriteLine($" Median: {Statistics.Percentile(annualPeaks, 0.50):F1} cfs"); // Dispersion Console.WriteLine("\nDispersion:"); -Console.WriteLine($" Range: {Statistics.Minimum(annualRainfall):F0} - {Statistics.Maximum(annualRainfall):F0} mm"); -Console.WriteLine($" Std Dev: {Statistics.StandardDeviation(annualRainfall):F1} mm"); -Console.WriteLine($" CV: {Statistics.CoefficientOfVariation(annualRainfall):P1}"); +Console.WriteLine($" Range: {Statistics.Minimum(annualPeaks):F0} - {Statistics.Maximum(annualPeaks):F0} cfs"); +Console.WriteLine($" Std Dev: {Statistics.StandardDeviation(annualPeaks):F1} cfs"); +Console.WriteLine($" CV: {Statistics.CoefficientOfVariation(annualPeaks):P1}"); // Shape Console.WriteLine("\nShape:"); -Console.WriteLine($" Skewness: {Statistics.Skewness(annualRainfall):F3}"); -Console.WriteLine($" Kurtosis: {Statistics.Kurtosis(annualRainfall):F3}"); - -// Percentiles -var percentiles = Statistics.Percentile(annualRainfall, new double[] { 10, 25, 50, 75, 90 }); -Console.WriteLine("\nPercentiles:"); -Console.WriteLine($" 10th: {percentiles[0]:F1} mm"); -Console.WriteLine($" 25th: {percentiles[1]:F1} mm"); -Console.WriteLine($" 50th: {percentiles[2]:F1} mm"); -Console.WriteLine($" 75th: {percentiles[3]:F1} mm"); -Console.WriteLine($" 90th: {percentiles[4]:F1} mm"); +Console.WriteLine($" Skewness: {Statistics.Skewness(annualPeaks):F4}"); +Console.WriteLine($" Kurtosis: {Statistics.Kurtosis(annualPeaks):F4}"); + +// L-Moments (more robust for hydrological data) +double[] lMoments = Statistics.LinearMoments(annualPeaks); +Console.WriteLine("\nL-Moments:"); +Console.WriteLine($" λ₁ (L-mean): {lMoments[0]:F1}"); +Console.WriteLine($" λ₂ (L-scale): {lMoments[1]:F1}"); +Console.WriteLine($" τ₃ (L-skew): {lMoments[2]:F4}"); +Console.WriteLine($" τ₄ (L-kurtosis):{lMoments[3]:F4}"); + +// Percentiles (five-number summary) +Console.WriteLine("\nFive-Number Summary:"); +Console.WriteLine($" Min: {Statistics.Minimum(annualPeaks):F0} cfs"); +Console.WriteLine($" Q1: {Statistics.Percentile(annualPeaks, 0.25):F0} cfs"); +Console.WriteLine($" Q2: {Statistics.Percentile(annualPeaks, 0.50):F0} cfs"); +Console.WriteLine($" Q3: {Statistics.Percentile(annualPeaks, 0.75):F0} cfs"); +Console.WriteLine($" Max: {Statistics.Maximum(annualPeaks):F0} cfs"); ``` ### Example 2: Comparing Two Datasets @@ -443,8 +625,8 @@ double sdBefore = Statistics.StandardDeviation(before); double sdAfter = Statistics.StandardDeviation(after); Console.WriteLine($"{"Std Dev",-20} | {sdBefore,10:F2} | {sdAfter,10:F2} | {sdAfter - sdBefore,10:F2}"); -double medBefore = Statistics.Percentile(before, 50); -double medAfter = Statistics.Percentile(after, 50); +double medBefore = Statistics.Percentile(before, 0.50); +double medAfter = Statistics.Percentile(after, 0.50); Console.WriteLine($"{"Median",-20} | {medBefore,10:F2} | {medAfter,10:F2} | {medAfter - medBefore,10:F2}"); // Effect size (Cohen's d) @@ -484,7 +666,29 @@ Console.WriteLine($" Winter (DJF): {winterMean:F0} cfs"); ## Running Statistics -For streaming data or very large datasets: +The `RunningStatistics` class implements Welford's online algorithm [[2]](#2) for numerically stable computation of variance and higher moments in a single pass through the data. This is essential for streaming data or very large datasets where it is impractical to store all values in memory. + +### Welford's Algorithm + +The classical two-pass approach (compute mean first, then deviations) requires two full scans of the data and storing the entire dataset. Welford's algorithm updates the running mean and sum of squared deviations incrementally with each new observation $x_n$: + +```math +\delta = x_n - M_1^{(n-1)} +``` + +```math +M_1^{(n)} = M_1^{(n-1)} + \frac{\delta}{n} +``` + +```math +M_2^{(n)} = M_2^{(n-1)} + \delta(x_n - M_1^{(n)}) +``` + +where $M_1$ is the running mean and $M_2$ is the running sum of squared deviations. The sample variance is then $s^2 = M_2 / (n-1)$. + +The `RunningStatistics` class extends this to the fourth moment ($M_3$ and $M_4$) for skewness and kurtosis computation. It also supports combining two `RunningStatistics` objects via the `Combine` method (or `+` operator), which is useful for parallel processing -- each thread can accumulate statistics independently and then merge results. + +**Available properties:** `Count`, `Minimum`, `Maximum`, `Mean`, `Variance`, `PopulationVariance`, `StandardDeviation`, `PopulationStandardDeviation`, `CoefficientOfVariation`, `Skewness`, `PopulationSkewness`, `Kurtosis`, `PopulationKurtosis`. ```cs using Numerics.Data.Statistics; @@ -525,6 +729,12 @@ Console.WriteLine($"Max: {runningStats.Maximum:F2}"); [1] Hosking, J. R. M. (1990). L-moments: Analysis and estimation of distributions using linear combinations of order statistics. *Journal of the Royal Statistical Society: Series B (Methodological)*, 52(1), 105-124. +[2] Welford, B. P. (1962). Note on a method for calculating corrected sums of squares and products. *Technometrics*, 4(3), 419-420. + +[3] Fisher, R. A. (1930). The moments of the distribution for normal samples of measures of departure from normality. *Proceedings of the Royal Society of London. Series A*, 130(812), 16-28. + +[4] Kendall, M. G. (1938). A new measure of rank correlation. *Biometrika*, 30(1/2), 81-93. + --- -[← Back to Index](../index.md) | [Next: Goodness-of-Fit →](goodness-of-fit.md) +[← Previous: Time Series](../data/time-series.md) | [Back to Index](../index.md) | [Next: Goodness-of-Fit →](goodness-of-fit.md) diff --git a/docs/statistics/goodness-of-fit.md b/docs/statistics/goodness-of-fit.md index bc713d18..2342b1ab 100644 --- a/docs/statistics/goodness-of-fit.md +++ b/docs/statistics/goodness-of-fit.md @@ -1,6 +1,6 @@ # Goodness-of-Fit Metrics -[← Previous: Descriptive Statistics](descriptive.md) | [Back to Index](../index.md) | [Next: MCMC Sampling →](../sampling/mcmc.md) +[← Previous: Descriptive Statistics](descriptive.md) | [Back to Index](../index.md) | [Next: Hypothesis Tests →](hypothesis-tests.md) Goodness-of-fit (GOF) metrics evaluate how well a statistical model fits observed data. The ***Numerics*** library provides comprehensive metrics for model selection, distribution fitting validation, and hydrological model evaluation. @@ -46,10 +46,15 @@ Console.WriteLine($" BIC: {bic:F2} (stronger penalty for parameters)"); ``` **Formulas:** + +```math +\text{AIC} = 2k - 2\ln(\hat{L}) +``` +```math +\text{AICc} = \text{AIC} + \frac{2k(k+1)}{n-k-1} ``` -AIC = 2k - 2·ln(L) -AICc = AIC + 2k(k+1)/(n-k-1) -BIC = k·ln(n) - 2·ln(L) +```math +\text{BIC} = k\ln(n) - 2\ln(\hat{L}) ``` Where: @@ -57,6 +62,52 @@ Where: - `n` = sample size - `L` = likelihood +#### AIC and Kullback-Leibler Divergence + +AIC is derived as an asymptotic approximation to the expected Kullback-Leibler (KL) divergence between the true data-generating model $f$ and the fitted candidate model $g$. The KL divergence measures the information lost when $g$ is used to approximate $f$: + +```math +D_{KL}(f \| g) = \int f(x) \log \frac{f(x)}{g(x|\hat{\theta})} \, dx +``` + +Akaike (1974) showed that the expected KL divergence can be estimated, up to a constant that is the same for all candidate models, by: + +```math +E[D_{KL}] \approx -2\log \hat{L} + 2k +``` + +Hence $\text{AIC} = 2k - 2\ln(\hat{L})$ estimates twice the expected information loss. The model with the lowest AIC is expected to be closest to the unknown true model in the KL sense. + +#### AICc Small-Sample Correction + +When the ratio $n/k < 40$, the standard AIC can exhibit substantial bias. The corrected AICc adds a finite-sample bias adjustment: + +```math +\text{AICc} = \text{AIC} + \frac{2k(k+1)}{n-k-1} +``` + +This correction term becomes negligible as $n$ grows large, so AICc converges to AIC for large samples. It is generally recommended to use AICc by default, since the correction is harmless when the sample is large. + +#### BIC and Bayesian Model Selection + +BIC approximates the log marginal likelihood (or evidence) for a model under certain regularity conditions. It can be interpreted as an approximation to the Bayes factor between a candidate model $M_i$ and a saturated model: + +```math +\text{BIC} \approx -2 \log P(\text{data} | M_i) + C +``` + +where $C$ is a constant common to all models. BIC penalizes model complexity more severely than AIC because the $\ln(n)$ term grows with sample size (compared to the constant penalty of 2 in AIC). As a result, BIC tends to prefer simpler models, especially for large $n$. + +#### Interpreting ΔAIC + +When comparing a set of candidate models, compute $\Delta_i = \text{AIC}_i - \text{AIC}_{\min}$ for each model. The following rules of thumb from Burnham and Anderson (2002) [[4]](#4) guide interpretation: + +| ΔAIC | Evidence for model | +|------|-------------------| +| 0--2 | Substantial support; model is competitive with the best | +| 4--7 | Considerably less support | +| > 10 | Essentially no support; model is implausible | + ### Comparing Multiple Models ```cs @@ -129,12 +180,13 @@ double rmse = GoodnessOfFit.RMSE(observed, gev); Console.WriteLine($"RMSE: {rmse:F2}"); -// With custom plotting positions -var plottingPos = PlottingPositions.Weibull(observed.Length); -double rmse2 = GoodnessOfFit.RMSE(observed, plottingPos, gev); +// With custom plotting positions (observed data must be sorted in ascending order) +var sortedObserved = observed.OrderBy(x => x).ToArray(); +var plottingPos = PlottingPositions.Weibull(sortedObserved.Length); +double rmse2 = GoodnessOfFit.RMSE(sortedObserved, plottingPos, gev); -// With parameter penalty -double rmse3 = GoodnessOfFit.RMSE(observed, gev.InverseCDF(plottingPos).ToArray(), k: gev.NumberOfParameters); +// With parameter penalty (both arrays must be in the same order) +double rmse3 = GoodnessOfFit.RMSE(sortedObserved, gev.InverseCDF(plottingPos).ToArray(), k: gev.NumberOfParameters); Console.WriteLine($"RMSE (Weibull plotting): {rmse2:F2}"); Console.WriteLine($"RMSE (with penalty): {rmse3:F2}"); @@ -205,15 +257,40 @@ else ``` **Formula:** -``` -NSE = 1 - Σ(O - M)² / Σ(O - Ō)² + +```math +\text{NSE} = 1 - \frac{\sum_{i=1}^{n}(O_i - M_i)^2}{\sum_{i=1}^{n}(O_i - \bar{O})^2} ``` Range: (-∞, 1], where 1 is perfect fit, 0 means model is as good as mean, <0 means worse than mean. +#### NSE Decomposition + +NSE can be decomposed into three interpretable components (Murphy, 1988 [[7]](#7)): + +```math +\text{NSE} = 2\alpha r - \alpha^2 - \beta^2 +``` + +where: +- $r$ = Pearson correlation coefficient between observed and modeled values +- $\alpha = \sigma_M / \sigma_O$ (ratio of standard deviations, measuring variability bias) +- $\beta = (\mu_M - \mu_O) / \sigma_O$ (normalized bias) + +This decomposition reveals whether poor NSE is caused by: +- **Poor correlation** ($r$ far from 1) -- indicates timing or phasing errors in the model +- **Wrong variability** ($\alpha$ far from 1) -- indicates the model over- or under-estimates the amplitude of fluctuations +- **Systematic bias** ($\beta$ far from 0) -- indicates consistent over- or under-prediction of the mean + ### Log Nash-Sutcliffe Efficiency -For better performance on low flows: +The Log-NSE variant applies a logarithmic transformation to both observed and modeled values before computing NSE. This gives more weight to low-flow performance, since the logarithm compresses large values and expands small values. The transformation is: + +```math +\text{Log-NSE} = 1 - \frac{\sum_{i=1}^{n}(\ln(O_i + \varepsilon) - \ln(M_i + \varepsilon))^2}{\sum_{i=1}^{n}(\ln(O_i + \varepsilon) - \overline{\ln(O + \varepsilon)})^2} +``` + +where $\varepsilon$ is a small constant (default: $\bar{O}/100$) added to prevent taking the logarithm of zero. Log-NSE is particularly useful for water quality assessment and ecological flow requirements, where accurate simulation of low-flow conditions is critical. ```cs double logNSE = GoodnessOfFit.LogNashSutcliffeEfficiency(observed, modeled); @@ -246,8 +323,9 @@ else ``` **Formula:** -``` -KGE = 1 - √[(r-1)² + (β-1)² + (γ-1)²] + +```math +\text{KGE} = 1 - \sqrt{(r-1)^2 + (\beta-1)^2 + (\gamma-1)^2} ``` Where: @@ -255,6 +333,35 @@ Where: - `β` = bias ratio (μ_modeled / μ_observed) - `γ` = variability ratio (CV_modeled / CV_observed) +#### KGE Component Interpretation + +The KGE decomposes model error into three orthogonal components, each diagnosing a distinct type of model deficiency: + +| Component | Meaning | Perfect Value | Diagnostic | +|-----------|---------|---------------|------------| +| $r$ | Pearson correlation | 1 | Timing and phasing of peaks and recessions | +| $\beta$ | Bias ratio ($\mu_M / \mu_O$) | 1 | Systematic over-estimation ($\beta > 1$) or under-estimation ($\beta < 1$) | +| Variability ratio | Spread of modeled vs. observed | 1 | Whether model reproduces the observed variability | + +**Original KGE (Gupta et al., 2009)** [[3]](#3): The `KlingGuptaEfficiency` method uses the standard deviation ratio $\alpha = \sigma_M / \sigma_O$ as the variability component: + +```math +\text{KGE} = 1 - \sqrt{(r-1)^2 + (\alpha - 1)^2 + (\beta - 1)^2} +``` + +**Modified KGE (Kling et al., 2012)** [[8]](#8): The `KlingGuptaEfficiencyMod` method replaces $\alpha$ with the coefficient of variation ratio $\gamma = \text{CV}_M / \text{CV}_O$, which avoids cross-correlation between the bias and variability components: + +```math +\text{KGE'} = 1 - \sqrt{(r-1)^2 + (\gamma - 1)^2 + (\beta - 1)^2} +``` + +where $\gamma = (\sigma_M / \mu_M) / (\sigma_O / \mu_O)$. The modified version is generally preferred because the bias ratio $\beta$ and the variability ratio $\gamma$ are mathematically independent, making the decomposition cleaner. + +**Diagnostic approach:** When KGE is low, examine which component dominates the Euclidean distance to guide model improvement: +- **Low $r$** -- improve model structure or parameterization to better capture temporal dynamics +- **$\beta \neq 1$** -- adjust bias correction or calibrate volume-related parameters +- **Variability ratio $\neq 1$** -- the model under- or over-estimates the spread; adjust parameters controlling flow variability + ### Percent Bias (PBIAS) ```cs @@ -273,9 +380,9 @@ else Console.WriteLine("Unsatisfactory (bias ≥ ±25%)"); if (pbias > 0) - Console.WriteLine("Model underestimates (positive bias)"); + Console.WriteLine("Model overestimates (positive bias)"); else if (pbias < 0) - Console.WriteLine("Model overestimates (negative bias)"); + Console.WriteLine("Model underestimates (negative bias)"); ``` ### RMSE-Observations Standard Deviation Ratio (RSR) @@ -356,6 +463,33 @@ Console.WriteLine($"sMAPE: {smape:F2}%"); ### Kolmogorov-Smirnov Test +The Kolmogorov-Smirnov (K-S) test statistic measures the maximum vertical distance between the empirical cumulative distribution function (ECDF) and the theoretical CDF: + +```math +D_n = \sup_x |F_n(x) - F(x)| +``` + +where $F_n(x)$ is the empirical CDF defined as the proportion of observations less than or equal to $x$: + +```math +F_n(x) = \frac{1}{n}\sum_{i=1}^{n} \mathbf{1}_{[x_i \leq x]} +``` + +and $F(x)$ is the CDF of the hypothesized distribution. + +In practice, since the ECDF is a step function, the supremum reduces to checking each order statistic. The implementation computes: + +```math +D_n = \max_{1 \leq i \leq n} \left\{ F(x_{i:n}) - \frac{i-1}{n}, \;\; \frac{i}{n} - F(x_{i:n}) \right\} +``` + +**Limitations:** +- Most sensitive near the center (median) of the distribution, less sensitive in the tails +- Critical values depend on whether parameters are estimated from data; when parameters are estimated, the Lilliefors correction is needed for valid p-values +- Conservative when used with estimated parameters (rejects too infrequently) +- Power decreases for heavy-tailed distributions +- The Anderson-Darling test is generally preferred when tail behavior is important + Tests if data comes from a specified distribution: ```cs @@ -375,6 +509,18 @@ Console.WriteLine("Smaller D indicates better fit"); ### Anderson-Darling Test +The Anderson-Darling (A-D) test statistic applies a weighting function that gives more emphasis to discrepancies in the tails of the distribution compared to the Kolmogorov-Smirnov test [[6]](#6). The test statistic is computed from the order statistics $x_{1:n} \leq x_{2:n} \leq \cdots \leq x_{n:n}$: + +```math +A^2 = -n - \sum_{i=1}^{n} \frac{2i-1}{n}\left[\ln F(x_{i:n}) + \ln(1 - F(x_{n+1-i:n}))\right] +``` + +where $F$ is the CDF of the hypothesized distribution. + +The implicit weighting function underlying the A-D statistic is $1/[F(x)(1-F(x))]$, which increases without bound as $F(x) \to 0$ or $F(x) \to 1$. This makes the test particularly sensitive to departures from the hypothesized distribution in the tails. For hydrological applications where tail behavior determines flood risk or drought severity, this tail sensitivity is a critical advantage over the K-S test. + +**Key advantage over K-S:** The tail weighting makes the Anderson-Darling test more powerful for detecting misfit in the extreme quantiles that matter most for risk analysis. + More sensitive to tail deviations: ```cs @@ -386,6 +532,21 @@ Console.WriteLine("Smaller A² indicates better fit"); ### Chi-Squared Test +The Chi-Squared ($\chi^2$) goodness-of-fit test compares observed bin frequencies to the frequencies expected under the hypothesized distribution. The data are divided into $k$ bins, and the test statistic is: + +```math +\chi^2 = \sum_{i=1}^{k} \frac{(O_i - E_i)^2}{E_i} +``` + +where $O_i$ is the observed frequency in bin $i$, and $E_i = n \cdot [F(b_i) - F(b_{i-1})]$ is the expected frequency computed from the hypothesized CDF $F$ evaluated at the bin boundaries $b_{i-1}$ and $b_i$. + +Under the null hypothesis, $\chi^2$ follows approximately a chi-squared distribution with $\text{df} = k - 1 - p$ degrees of freedom, where $p$ is the number of parameters estimated from the data. + +**Rules of thumb for bin selection:** +- Minimum expected frequency per bin should be at least 5 for the chi-squared approximation to be valid +- Too few bins reduces the power of the test; too many bins makes the test statistic unstable +- The implementation uses a default histogram binning; for formal hypothesis testing, consider the sensitivity of results to bin choice + For discrete or binned data: ```cs @@ -423,14 +584,28 @@ Console.WriteLine($" Balanced Accuracy: {balancedAcc:P1}"); ### Example 1: Complete Distribution Comparison +This example compares candidate distributions for the White River near Nora, Indiana. Results can be cross-checked with the worked examples in Rao & Hamed (2000), Chapter 7. + +**Data source:** Rao, A. R. & Hamed, K. H. (2000). *Flood Frequency Analysis*. CRC Press, Table 7.1.2. +See also: [`example-data/white-river-nora-floods.csv`](../example-data/white-river-nora-floods.csv) + ```cs using Numerics.Data.Statistics; using Numerics.Distributions; -double[] annualPeaks = { 12500, 15300, 11200, 18700, 14100, 16800, 13400, 17200, 10500, 19300 }; +// White River near Nora, IN — 62 years of annual peak streamflow (cfs) +double[] annualPeaks = { + 23200, 2950, 10300, 23200, 4540, 9960, 10800, 26900, 23300, 20400, + 8480, 3150, 9380, 32400, 20800, 11100, 7270, 9600, 14600, 14300, + 22500, 14700, 12700, 9740, 3050, 8830, 12000, 30400, 27000, 15200, + 8040, 11700, 20300, 22700, 30400, 9180, 4870, 14700, 12800, 13700, + 7960, 9830, 12500, 10700, 13200, 14700, 14300, 4050, 14600, 14400, + 19200, 7160, 12100, 8650, 10600, 24500, 14400, 6300, 9560, 15800, + 14300, 28700 +}; // Candidate distributions -var candidates = new (string Name, IUnivariateDistribution Dist)[] +var candidates = new (string Name, UnivariateDistributionBase Dist)[] { ("LP3", new LogPearsonTypeIII()), ("GEV", new GeneralizedExtremeValue()), @@ -446,7 +621,7 @@ var results = new List<(string Name, double AIC, double BIC, double RMSE, double foreach (var (name, dist) in candidates) { // Fit distribution - dist.Estimate(annualPeaks, ParameterEstimationMethod.MethodOfLinearMoments); + ((IEstimation)dist).Estimate(annualPeaks, ParameterEstimationMethod.MethodOfLinearMoments); // Compute metrics double logLik = annualPeaks.Sum(x => dist.LogPDF(x)); @@ -571,6 +746,35 @@ foreach (var (name, modeled) in models) } ``` +## When to Use Which Metric + +Selecting the right goodness-of-fit metric depends on the analysis goal. The following table summarizes recommended metrics for common tasks, all of which are available in the `GoodnessOfFit` class: + +| Goal | Recommended Metric | Notes | +|------|-------------------|-------| +| Model selection (different number of parameters) | AIC / AICc / BIC | Information criteria; lower is better | +| Model selection (same number of parameters) | RMSE or NSE | Direct comparison of fit quality | +| Distribution fit validation | K-S, A-D, $\chi^2$ | A-D preferred for tail sensitivity | +| Hydrological model calibration | KGE | Decomposes error into correlation, bias, and variability | +| Hydrological model validation | NSE + PBIAS + RSR | Multiple complementary metrics recommended (Moriasi et al., 2007) [[2]](#2) | +| Forecast accuracy | MAPE / sMAPE | Scale-independent percentage errors | +| Water balance assessment | VE + PBIAS | Volume-focused metrics | +| Low-flow emphasis | Log-NSE | Log transform gives more weight to low values | +| Classification problems | F1 Score, Balanced Accuracy | When output is binary (0/1) | + +### Moriasi et al. (2007) Performance Ratings + +The following performance ratings from Moriasi et al. (2007) [[2]](#2) are widely used in watershed modeling to evaluate simulation results: + +| Performance | NSE | RSR | PBIAS (streamflow) | +|------------|-----|-----|-------------------| +| Very Good | > 0.75 | 0.00--0.50 | < +/-10% | +| Good | 0.65--0.75 | 0.50--0.60 | +/-10--15% | +| Satisfactory | 0.50--0.65 | 0.60--0.70 | +/-15--25% | +| Unsatisfactory | < 0.50 | > 0.70 | >= +/-25% | + +These thresholds apply to streamflow simulations at a monthly time step. For other constituents (e.g., sediment, nutrients) or sub-monthly time steps, the thresholds may be relaxed. No single metric fully characterizes model performance; using multiple complementary metrics provides a more complete assessment. + ## Best Practices 1. **Use multiple metrics** - No single metric captures all aspects of fit @@ -591,6 +795,16 @@ foreach (var (name, modeled) in models) [3] Gupta, H. V., Kling, H., Yilmaz, K. K., & Martinez, G. F. (2009). Decomposition of the mean squared error and NSE performance criteria: Implications for improving hydrological modelling. *Journal of Hydrology*, 377(1-2), 80-91. +[4] Burnham, K. P., & Anderson, D. R. (2002). *Model Selection and Multimodel Inference: A Practical Information-Theoretic Approach* (2nd ed.). Springer. + +[5] Schwarz, G. (1978). Estimating the dimension of a model. *Annals of Statistics*, 6(2), 461-464. + +[6] Anderson, T. W., & Darling, D. A. (1954). A test of goodness of fit. *Journal of the American Statistical Association*, 49(268), 765-769. + +[7] Murphy, A. H. (1988). Skill scores based on the mean square error and their relationships to the correlation coefficient. *Monthly Weather Review*, 116(12), 2417-2424. + +[8] Kling, H., Fuchs, M., & Paulin, M. (2012). Runoff conditions in the upper Danube basin under an ensemble of climate change scenarios. *Journal of Hydrology*, 424-425, 264-277. + --- -[← Previous: Descriptive Statistics](descriptive.md) | [Back to Index](../index.md) | [Next: MCMC Sampling →](../sampling/mcmc.md) +[← Previous: Descriptive Statistics](descriptive.md) | [Back to Index](../index.md) | [Next: Hypothesis Tests →](hypothesis-tests.md) diff --git a/docs/statistics/hypothesis-tests.md b/docs/statistics/hypothesis-tests.md index bbe25622..b5c8573f 100644 --- a/docs/statistics/hypothesis-tests.md +++ b/docs/statistics/hypothesis-tests.md @@ -1,14 +1,32 @@ # Hypothesis Tests -[← Previous: Goodness-of-Fit](goodness-of-fit.md) | [Back to Index](../index.md) | [Next: Convergence Diagnostics →](../sampling/convergence-diagnostics.md) +[← Previous: Goodness-of-Fit](goodness-of-fit.md) | [Back to Index](../index.md) | [Next: Univariate Distributions →](../distributions/univariate.md) -The ***Numerics*** library provides statistical hypothesis tests for comparing samples, testing distributions, and detecting trends. These tests are essential for data analysis, model validation, and quality control. +Statistical hypothesis testing is a method of statistical inference used to decide whether observed data sufficiently support a particular hypothesis. The ***Numerics*** library provides a comprehensive set of hypothesis tests that return **p-values** for making statistical decisions. If the p-value is less than the chosen significance level (typically α = 0.05), the null hypothesis is rejected in favor of the alternative hypothesis. + +## Understanding p-values + +All hypothesis tests in ***Numerics*** return **p-values**, not test statistics. The p-value represents the probability of obtaining results at least as extreme as the observed data, assuming the null hypothesis is true. + +**Decision rule:** +- p < 0.01: Very strong evidence against H₀ +- p < 0.05: Strong evidence against H₀ +- p < 0.10: Weak evidence against H₀ +- p ≥ 0.10: Insufficient evidence to reject H₀ ## t-Tests +The t-test compares sample means using the **t-statistic**, which measures how many standard errors the observed mean is from the hypothesized value. Under the null hypothesis, this statistic follows a Student's t-distribution. + ### One-Sample t-Test -Test if sample mean differs from hypothesized value: +Tests if the sample mean differs from a hypothesized population mean. The test statistic is: + +```math +t = \frac{\bar{x} - \mu_0}{s / \sqrt{n}} +``` + +where $\bar{x}$ is the sample mean, $\mu_0$ is the hypothesized mean, $s$ is the sample standard deviation, and $n$ is the sample size. Under $H_0$, $t \sim t_{n-1}$. ```cs using Numerics.Data.Statistics; @@ -16,18 +34,19 @@ using Numerics.Data.Statistics; double[] sample = { 12.5, 13.2, 11.8, 14.1, 12.9, 13.5, 12.2, 13.8 }; double mu0 = 12.0; // Hypothesized mean -// Compute t-statistic -double t = HypothesisTests.OneSampleTtest(sample, mu0); +// Returns the two-sided p-value +double pValue = HypothesisTests.OneSampleTtest(sample, mu0); Console.WriteLine($"One-sample t-test:"); -Console.WriteLine($" t-statistic: {t:F3}"); Console.WriteLine($" Sample mean: {Statistics.Mean(sample):F2}"); Console.WriteLine($" Hypothesized mean: {mu0:F2}"); +Console.WriteLine($" p-value: {pValue:F4}"); +Console.WriteLine($" Degrees of freedom: {sample.Length - 1}"); -// Compare with critical value -int df = sample.Length - 1; -Console.WriteLine($" Degrees of freedom: {df}"); -Console.WriteLine(" If |t| > t_critical (e.g., 2.365 for α=0.05, df=7), reject H₀"); +if (pValue < 0.05) + Console.WriteLine(" Result: Reject H₀ - mean differs significantly from 12.0"); +else + Console.WriteLine(" Result: Fail to reject H₀ - insufficient evidence of difference"); ``` **Hypotheses:** @@ -38,20 +57,29 @@ Console.WriteLine(" If |t| > t_critical (e.g., 2.365 for α=0.05, df=7), reject #### Equal Variance (Pooled) t-Test +Tests if means of two independent samples are equal, assuming equal variances. The test uses a **pooled variance** estimate $s_p^2 = \frac{(n_1-1)s_1^2 + (n_2-1)s_2^2}{n_1 + n_2 - 2}$ and the statistic: + +```math +t = \frac{\bar{x}_1 - \bar{x}_2}{s_p\sqrt{1/n_1 + 1/n_2}}, \quad t \sim t_{n_1+n_2-2} +``` + ```cs double[] sample1 = { 12.5, 13.2, 11.8, 14.1, 12.9 }; double[] sample2 = { 15.3, 14.8, 15.9, 14.5, 15.1 }; -// Test if means are equal (assuming equal variances) -double t = HypothesisTests.EqualVarianceTtest(sample1, sample2); +// Returns the two-sided p-value +double pValue = HypothesisTests.EqualVarianceTtest(sample1, sample2); Console.WriteLine($"Equal variance t-test:"); -Console.WriteLine($" t-statistic: {t:F3}"); Console.WriteLine($" Sample 1 mean: {Statistics.Mean(sample1):F2}"); Console.WriteLine($" Sample 2 mean: {Statistics.Mean(sample2):F2}"); +Console.WriteLine($" p-value: {pValue:F4}"); +Console.WriteLine($" Degrees of freedom: {sample1.Length + sample2.Length - 2}"); -int df = sample1.Length + sample2.Length - 2; -Console.WriteLine($" Degrees of freedom: {df}"); +if (pValue < 0.05) + Console.WriteLine(" Result: Significant difference between means"); +else + Console.WriteLine(" Result: No significant difference between means"); ``` **Hypotheses:** @@ -60,58 +88,71 @@ Console.WriteLine($" Degrees of freedom: {df}"); #### Unequal Variance (Welch's) t-Test +Use when variances may be different between groups: + ```cs -// Test if means are equal (not assuming equal variances) -double t_welch = HypothesisTests.UnequalVarianceTtest(sample1, sample2); +// Returns the two-sided p-value (Welch's approximation for df) +double pValue = HypothesisTests.UnequalVarianceTtest(sample1, sample2); -Console.WriteLine($"\nWelch's t-test:"); -Console.WriteLine($" t-statistic: {t_welch:F3}"); -Console.WriteLine(" Use when variances appear different"); +Console.WriteLine($"Welch's t-test (unequal variances):"); +Console.WriteLine($" p-value: {pValue:F4}"); +Console.WriteLine(" Use when variances appear different between groups"); ``` #### Paired t-Test -For before/after or matched pairs: +For matched pairs or before/after comparisons: ```cs double[] before = { 120, 135, 118, 142, 128 }; double[] after = { 115, 130, 112, 138, 125 }; -// Test if treatment had effect -double t_paired = HypothesisTests.PairedTtest(before, after); +// Returns the two-sided p-value +double pValue = HypothesisTests.PairedTtest(before, after); Console.WriteLine($"Paired t-test:"); -Console.WriteLine($" t-statistic: {t_paired:F3}"); -Console.WriteLine($" Mean difference: {Statistics.Mean(before) - Statistics.Mean(after):F2}"); - -// Differences -double[] diffs = new double[before.Length]; -for (int i = 0; i < before.Length; i++) - diffs[i] = before[i] - after[i]; +Console.WriteLine($" Mean before: {Statistics.Mean(before):F1}"); +Console.WriteLine($" Mean after: {Statistics.Mean(after):F1}"); +Console.WriteLine($" Mean difference: {Statistics.Mean(before) - Statistics.Mean(after):F1}"); +Console.WriteLine($" p-value: {pValue:F4}"); -Console.WriteLine($" SE of difference: {Statistics.StandardDeviation(diffs) / Math.Sqrt(diffs.Length):F2}"); +if (pValue < 0.05) + Console.WriteLine(" Result: Treatment had a significant effect"); +else + Console.WriteLine(" Result: No significant treatment effect detected"); ``` **Hypotheses:** - H₀: μ_diff = 0 - H₁: μ_diff ≠ 0 -## F-Test +## F-Test for Variance Equality + +Tests if two populations have equal variances. The test statistic is the ratio of sample variances: -Test equality of variances: +```math +F = \frac{s_1^2}{s_2^2}, \quad F \sim F_{n_1-1, n_2-1} +``` + +Values far from 1 (either large or small) suggest unequal variances. ```cs double[] sample1 = { 10, 12, 14, 16, 18 }; double[] sample2 = { 11, 13, 15, 17, 19, 21, 23 }; -// Test if variances are equal -double f = HypothesisTests.Ftest(sample1, sample2); +// Returns the p-value +double pValue = HypothesisTests.Ftest(sample1, sample2); Console.WriteLine($"F-test for equal variances:"); -Console.WriteLine($" F-statistic: {f:F3}"); Console.WriteLine($" Variance 1: {Statistics.Variance(sample1):F2}"); Console.WriteLine($" Variance 2: {Statistics.Variance(sample2):F2}"); +Console.WriteLine($" p-value: {pValue:F4}"); Console.WriteLine($" df1 = {sample1.Length - 1}, df2 = {sample2.Length - 1}"); + +if (pValue < 0.05) + Console.WriteLine(" Result: Variances are significantly different"); +else + Console.WriteLine(" Result: No significant difference in variances"); ``` **Hypotheses:** @@ -120,16 +161,16 @@ Console.WriteLine($" df1 = {sample1.Length - 1}, df2 = {sample2.Length - 1}"); ### F-Test for Nested Models -Compare restricted and full models: +Compares restricted and full regression models: ```cs -// Example: Testing if additional predictors improve model +// SSE values from model fitting double sseRestricted = 150.0; // SSE of restricted model double sseFull = 120.0; // SSE of full model int dfRestricted = 47; // n - k_restricted - 1 int dfFull = 45; // n - k_full - 1 -HypothesisTests.FtestModels(sseRestricted, sseFull, dfRestricted, dfFull, +HypothesisTests.FtestModels(sseRestricted, sseFull, dfRestricted, dfFull, out double fStat, out double pValue); Console.WriteLine($"F-test for model comparison:"); @@ -137,78 +178,90 @@ Console.WriteLine($" F-statistic: {fStat:F3}"); Console.WriteLine($" p-value: {pValue:F4}"); if (pValue < 0.05) - Console.WriteLine(" Reject H₀: Full model is significantly better"); + Console.WriteLine(" Result: Full model is significantly better"); else - Console.WriteLine(" Fail to reject H₀: Models not significantly different"); + Console.WriteLine(" Result: Models not significantly different"); ``` -## Normality Tests +## Normality Test ### Jarque-Bera Test -Tests if data follows normal distribution using skewness and kurtosis: +Tests if data follows a normal distribution by measuring departures from the skewness ($S=0$) and excess kurtosis ($K=0$) expected under normality. The test statistic is: + +```math +JB = \frac{n}{6}\left(S^2 + \frac{K^2}{4}\right) +``` + +where $S$ is the sample skewness and $K$ is the sample excess kurtosis. Under $H_0$, $JB \sim \chi^2_2$ asymptotically. ```cs double[] data = { 10.5, 12.3, 11.8, 15.2, 13.7, 14.1, 16.8, 12.9, 11.2, 14.5 }; -// Test for normality -double jb = HypothesisTests.JarqueBeraTest(data); +// Returns the p-value (chi-squared with df=2) +double pValue = HypothesisTests.JarqueBeraTest(data); Console.WriteLine($"Jarque-Bera normality test:"); -Console.WriteLine($" JB statistic: {jb:F3}"); Console.WriteLine($" Skewness: {Statistics.Skewness(data):F3}"); Console.WriteLine($" Kurtosis: {Statistics.Kurtosis(data):F3}"); -Console.WriteLine(" Critical value (α=0.05): 5.99 (χ² with df=2)"); +Console.WriteLine($" p-value: {pValue:F4}"); -if (jb < 5.99) - Console.WriteLine(" Fail to reject H₀: Data appears normally distributed"); +if (pValue < 0.05) + Console.WriteLine(" Result: Data is not normally distributed"); else - Console.WriteLine(" Reject H₀: Data not normally distributed"); + Console.WriteLine(" Result: Cannot reject normality assumption"); ``` **Hypotheses:** - H₀: Data is normally distributed -- H₁: Data is not normal +- H₁: Data is not normally distributed -## Randomness Tests +## Randomness and Independence Tests ### Wald-Wolfowitz Runs Test -Tests if sequence is random: +Tests if a sequence is random (tests for independence and stationarity): ```cs double[] sequence = { 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0 }; -double z = HypothesisTests.WaldWolfowitzTest(sequence); +// Returns the two-sided p-value +double pValue = HypothesisTests.WaldWolfowitzTest(sequence); Console.WriteLine($"Wald-Wolfowitz runs test:"); -Console.WriteLine($" z-statistic: {z:F3}"); +Console.WriteLine($" p-value: {pValue:F4}"); -if (Math.Abs(z) > 1.96) - Console.WriteLine(" Reject H₀: Sequence is not random"); +if (pValue < 0.05) + Console.WriteLine(" Result: Sequence is not random"); else - Console.WriteLine(" Fail to reject H₀: Sequence appears random"); + Console.WriteLine(" Result: Cannot reject randomness assumption"); ``` ### Ljung-Box Test -Tests for autocorrelation in time series: +Tests for autocorrelation in time series data. The test statistic sums squared autocorrelation coefficients: + +```math +Q = n(n+2)\sum_{k=1}^{m}\frac{\hat{\rho}_k^2}{n-k} +``` + +where $\hat{\rho}_k$ is the sample autocorrelation at lag $k$, $n$ is the sample size, and $m$ is the number of lags tested. Under $H_0$ (no autocorrelation), $Q \sim \chi^2_m$. ```cs double[] timeSeries = { 12.5, 13.2, 11.8, 14.1, 12.9, 13.5, 12.2, 13.8, 14.5, 13.1 }; int lagMax = 5; // Test lags 1 through 5 -double q = HypothesisTests.LjungBoxTest(timeSeries, lagMax); +// Returns the p-value (chi-squared with df = lagMax) +double pValue = HypothesisTests.LjungBoxTest(timeSeries, lagMax); Console.WriteLine($"Ljung-Box test for autocorrelation:"); -Console.WriteLine($" Q-statistic: {q:F3}"); Console.WriteLine($" Lags tested: {lagMax}"); -Console.WriteLine($" Critical value (α=0.05, df={lagMax}): ~11.07"); +Console.WriteLine($" p-value: {pValue:F4}"); -if (q > 11.07) - Console.WriteLine(" Reject H₀: Significant autocorrelation present"); +if (pValue < 0.05) + Console.WriteLine(" Result: Significant autocorrelation detected"); else - Console.WriteLine(" Fail to reject H₀: No significant autocorrelation"); + Console.WriteLine(" Result: No significant autocorrelation"); ``` **Hypotheses:** @@ -219,18 +272,25 @@ else ### Mann-Whitney U Test -Non-parametric alternative to two-sample t-test: +Non-parametric alternative to two-sample t-test (tests if distributions differ): ```cs -double[] group1 = { 12, 15, 18, 21, 24 }; -double[] group2 = { 10, 13, 16, 19, 22, 25 }; +double[] group1 = { 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42 }; +double[] group2 = { 10, 13, 16, 19, 22, 25, 28, 31, 34, 37, 40 }; -double u = HypothesisTests.MannWhitneyTest(group1, group2); +// Note: First sample must have length ≤ second sample +// Combined samples must have length > 20 (here: 11 + 11 = 22) +// Returns the two-sided p-value +double pValue = HypothesisTests.MannWhitneyTest(group1, group2); Console.WriteLine($"Mann-Whitney U test:"); -Console.WriteLine($" U-statistic: {u:F3}"); -Console.WriteLine(" Tests if distributions are different (rank-based)"); -Console.WriteLine(" No assumption of normality required"); +Console.WriteLine($" p-value: {pValue:F4}"); +Console.WriteLine(" (Rank-based, no normality assumption required)"); + +if (pValue < 0.05) + Console.WriteLine(" Result: Distributions differ significantly"); +else + Console.WriteLine(" Result: No significant difference in distributions"); ``` **Hypotheses:** @@ -241,22 +301,28 @@ Console.WriteLine(" No assumption of normality required"); ### Mann-Kendall Trend Test -Detects monotonic trends in time series: +Detects monotonic trends in time series (non-parametric). The test statistic $S$ counts concordant minus discordant pairs: + +```math +S = \sum_{i=1}^{n-1}\sum_{j=i+1}^{n} \text{sgn}(x_j - x_i) +``` + +where $\text{sgn}(x) = 1$ if $x > 0$, $-1$ if $x < 0$, and $0$ if $x = 0$. For $n \geq 10$, $S$ is approximately normal with variance $\text{Var}(S) = \frac{n(n-1)(2n+5)}{18}$, and the standardized statistic $Z = S/\sqrt{\text{Var}(S)}$ is used to compute the p-value. A positive $S$ indicates an upward trend; negative indicates downward. ```cs double[] timeSeries = { 10, 12, 11, 15, 14, 18, 17, 21, 20, 24 }; -double s = HypothesisTests.MannKendallTest(timeSeries); +// Requires sample size ≥ 10 +// Returns the two-sided p-value +double pValue = HypothesisTests.MannKendallTest(timeSeries); Console.WriteLine($"Mann-Kendall trend test:"); -Console.WriteLine($" S-statistic: {s:F3}"); +Console.WriteLine($" p-value: {pValue:F4}"); -if (s > 1.96) - Console.WriteLine(" Significant increasing trend detected"); -else if (s < -1.96) - Console.WriteLine(" Significant decreasing trend detected"); +if (pValue < 0.05) + Console.WriteLine(" Result: Significant monotonic trend detected"); else - Console.WriteLine(" No significant trend"); + Console.WriteLine(" Result: No significant trend"); ``` **Hypotheses:** @@ -265,190 +331,247 @@ else ### Linear Trend Test -Tests for linear relationship: +Tests for significant linear relationship (parametric): ```cs double[] time = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; double[] values = { 10.5, 11.2, 12.8, 13.1, 14.5, 15.2, 16.1, 16.8, 17.5, 18.2 }; -double t = HypothesisTests.LinearTrendTest(time, values); +// Returns the two-sided p-value +double pValue = HypothesisTests.LinearTrendTest(time, values); Console.WriteLine($"Linear trend test:"); -Console.WriteLine($" t-statistic: {t:F3}"); -Console.WriteLine(" Tests if slope is significantly different from zero"); +Console.WriteLine($" p-value: {pValue:F4}"); -if (Math.Abs(t) > 2.306) // Critical value for df=8, α=0.05 - Console.WriteLine(" Significant linear trend detected"); +if (pValue < 0.05) + Console.WriteLine(" Result: Significant linear trend detected"); else - Console.WriteLine(" No significant linear trend"); + Console.WriteLine(" Result: No significant linear trend"); ``` ## Unimodality Test -Tests if distribution has single peak: +Tests if distribution has a single peak using Gaussian Mixture Model comparison: ```cs double[] data = { 10, 11, 12, 13, 14, 15, 14, 13, 12, 11, 10 }; -double u = HypothesisTests.UnimodalityTest(data); +// Requires sample size ≥ 10 +// Returns the p-value (likelihood ratio test with chi-squared df=3) +double pValue = HypothesisTests.UnimodalityTest(data); Console.WriteLine($"Unimodality test:"); -Console.WriteLine($" Test statistic: {u:F3}"); +Console.WriteLine($" p-value: {pValue:F4}"); -if (u < -1.96) - Console.WriteLine(" Reject H₀: Data is multimodal"); +if (pValue < 0.05) + Console.WriteLine(" Result: Evidence of multimodality (bimodal or more)"); else - Console.WriteLine(" Fail to reject H₀: Data appears unimodal"); + Console.WriteLine(" Result: Cannot reject unimodality"); ``` -## Practical Examples +## Hydrological Applications + +### Example 1: Testing for Stationarity in Annual Maximum Floods -### Example 1: Comparing Treatment Groups +A fundamental assumption in flood frequency analysis is that the annual maximum flood series is stationary (no trend over time). This example demonstrates how to test this assumption: ```cs using Numerics.Data.Statistics; +using Numerics.Distributions; -// Control and treatment groups -double[] control = { 120, 135, 118, 142, 128, 133, 125, 138 }; -double[] treatment = { 115, 125, 110, 130, 120, 128, 118, 130 }; - -Console.WriteLine("Treatment Comparison Study"); -Console.WriteLine("=" + new string('=', 50)); - -// Descriptive statistics -Console.WriteLine($"\nControl: mean={Statistics.Mean(control):F1}, " + - $"SD={Statistics.StandardDeviation(control):F1}"); -Console.WriteLine($"Treatment: mean={Statistics.Mean(treatment):F1}, " + - $"SD={Statistics.StandardDeviation(treatment):F1}"); +// Annual maximum peak flows (cfs) from 1990-2019 +double[] annualMaxFlows = { + 12500, 15300, 11200, 18700, 14100, 16800, 13400, 17200, 10500, 19300, + 14800, 16200, 13100, 18500, 15600, 17800, 12800, 19100, 14300, 16500, + 13800, 18200, 15100, 17400, 12300, 19800, 14600, 16900, 13500, 18900 +}; -// Test for equal variances -double f = HypothesisTests.Ftest(control, treatment); -Console.WriteLine($"\nF-test for equal variances: F={f:F3}"); +double[] years = Enumerable.Range(1990, annualMaxFlows.Length).Select(y => (double)y).ToArray(); -// Choose appropriate t-test -double t; -if (f < 4.0) // Approximate F-critical value +Console.WriteLine("Stationarity Analysis of Annual Maximum Floods"); +Console.WriteLine("=" + new string('=', 50)); +Console.WriteLine($"Record length: {annualMaxFlows.Length} years (1990-2019)"); +Console.WriteLine($"Mean: {Statistics.Mean(annualMaxFlows):F0} cfs"); +Console.WriteLine($"Std Dev: {Statistics.StandardDeviation(annualMaxFlows):F0} cfs"); + +// Test 1: Mann-Kendall test for monotonic trend +double pMK = HypothesisTests.MannKendallTest(annualMaxFlows); +Console.WriteLine($"\nMann-Kendall Trend Test:"); +Console.WriteLine($" p-value: {pMK:F4}"); +Console.WriteLine($" Result: {(pMK < 0.05 ? "Trend detected - stationarity violated" : "No significant trend")}"); + +// Test 2: Linear trend test +double pLinear = HypothesisTests.LinearTrendTest(years, annualMaxFlows); +Console.WriteLine($"\nLinear Trend Test:"); +Console.WriteLine($" p-value: {pLinear:F4}"); +Console.WriteLine($" Result: {(pLinear < 0.05 ? "Linear trend detected" : "No significant linear trend")}"); + +// Test 3: Ljung-Box test for serial correlation +double pLB = HypothesisTests.LjungBoxTest(annualMaxFlows, lagMax: 5); +Console.WriteLine($"\nLjung-Box Autocorrelation Test (lag=5):"); +Console.WriteLine($" p-value: {pLB:F4}"); +Console.WriteLine($" Result: {(pLB < 0.05 ? "Autocorrelation detected" : "No significant autocorrelation")}"); + +// Overall assessment +Console.WriteLine("\n" + new string('-', 50)); +bool isStationary = pMK >= 0.05 && pLinear >= 0.05 && pLB >= 0.05; +if (isStationary) { - t = HypothesisTests.EqualVarianceTtest(control, treatment); - Console.WriteLine($"Equal variance t-test: t={t:F3}"); + Console.WriteLine("Assessment: Data appears stationary - proceed with frequency analysis"); } else { - t = HypothesisTests.UnequalVarianceTtest(control, treatment); - Console.WriteLine($"Unequal variance t-test: t={t:F3}"); + Console.WriteLine("Assessment: Potential non-stationarity detected"); + Console.WriteLine("Consider: detrending, using shorter record, or non-stationary methods"); } - -if (Math.Abs(t) > 2.145) // Approximate critical value - Console.WriteLine("Conclusion: Significant difference detected (p < 0.05)"); -else - Console.WriteLine("Conclusion: No significant difference (p ≥ 0.05)"); ``` -### Example 2: Time Series Analysis - -```cs -double[] monthlyData = { 125, 130, 135, 132, 138, 145, 142, 148, 155, 152, 158, 165 }; - -Console.WriteLine("Time Series Analysis"); -Console.WriteLine("=" + new string('=', 50)); +### Example 2: Comparing Flood Records Between Two Periods -// Test for trend -double[] months = Enumerable.Range(1, monthlyData.Length).Select(i => (double)i).ToArray(); -double tTrend = HypothesisTests.LinearTrendTest(months, monthlyData); +Test whether flood characteristics have changed between historical and recent periods: -Console.WriteLine($"\nLinear trend test: t={tTrend:F3}"); -if (Math.Abs(tTrend) > 2.228) // Critical value for df=10 - Console.WriteLine("Significant trend detected"); - -// Test for autocorrelation -double q = HypothesisTests.LjungBoxTest(monthlyData, lagMax: 3); - -Console.WriteLine($"\nLjung-Box test (lag 3): Q={q:F3}"); -if (q > 7.815) // Chi-squared critical value - Console.WriteLine("Significant autocorrelation detected"); +```cs +// Split record into two periods +double[] period1 = { 12500, 15300, 11200, 18700, 14100, 16800, 13400, 17200, 10500, 19300 }; // 1990-1999 +double[] period2 = { 14800, 16200, 13100, 18500, 15600, 17800, 12800, 19100, 14300, 16500 }; // 2000-2009 + +Console.WriteLine("Comparison of Flood Characteristics: 1990s vs 2000s"); +Console.WriteLine("=" + new string('=', 55)); + +Console.WriteLine($"\nPeriod 1 (1990-1999):"); +Console.WriteLine($" Mean: {Statistics.Mean(period1):F0} cfs"); +Console.WriteLine($" Std Dev: {Statistics.StandardDeviation(period1):F0} cfs"); + +Console.WriteLine($"\nPeriod 2 (2000-2009):"); +Console.WriteLine($" Mean: {Statistics.Mean(period2):F0} cfs"); +Console.WriteLine($" Std Dev: {Statistics.StandardDeviation(period2):F0} cfs"); + +// Test for difference in means +double pMean = HypothesisTests.EqualVarianceTtest(period1, period2); +Console.WriteLine($"\nt-test for difference in means:"); +Console.WriteLine($" p-value: {pMean:F4}"); +Console.WriteLine($" Result: {(pMean < 0.05 ? "Means differ significantly" : "No significant difference in means")}"); + +// Test for difference in variances +double pVar = HypothesisTests.Ftest(period1, period2); +Console.WriteLine($"\nF-test for difference in variances:"); +Console.WriteLine($" p-value: {pVar:F4}"); +Console.WriteLine($" Result: {(pVar < 0.05 ? "Variances differ significantly" : "No significant difference in variances")}"); + +// Interpretation for flood frequency analysis +Console.WriteLine("\n" + new string('-', 55)); +if (pMean >= 0.05 && pVar >= 0.05) + Console.WriteLine("Conclusion: Records can be combined for frequency analysis"); else - Console.WriteLine("No significant autocorrelation"); - -// Mann-Kendall for monotonic trend -double s = HypothesisTests.MannKendallTest(monthlyData); - -Console.WriteLine($"\nMann-Kendall test: S={s:F3}"); -if (s > 1.96) - Console.WriteLine("Significant increasing trend (non-parametric)"); + Console.WriteLine("Conclusion: Consider analyzing periods separately or investigating causes"); ``` -### Example 3: Quality Control +### Example 3: Testing Normality of Log-Transformed Flood Data + +Log-Pearson Type III analysis assumes the log-transformed data follows a Pearson Type III distribution. Testing normality of log-transformed data can indicate if a simpler Log-Normal model might suffice: ```cs -// Historical process mean -double mu0 = 50.0; +double[] annualPeaks = { 12500, 15300, 11200, 18700, 14100, 16800, 13400, 17200, 10500, 19300, + 14800, 16200, 13100, 18500, 15600, 17800, 12800, 19100, 14300, 16500 }; -// New sample -double[] newSample = { 51.2, 52.1, 49.8, 51.5, 50.9, 52.3, 51.0, 50.5 }; +// Log-transform the data +double[] logPeaks = annualPeaks.Select(x => Math.Log10(x)).ToArray(); -Console.WriteLine("Quality Control Check"); -Console.WriteLine("=" + new string('=', 50)); +Console.WriteLine("Normality Test for Log-Transformed Annual Peak Flows"); +Console.WriteLine("=" + new string('=', 55)); + +Console.WriteLine($"\nLog-transformed data statistics:"); +Console.WriteLine($" Mean: {Statistics.Mean(logPeaks):F4}"); +Console.WriteLine($" Std Dev: {Statistics.StandardDeviation(logPeaks):F4}"); +Console.WriteLine($" Skewness: {Statistics.Skewness(logPeaks):F4}"); +Console.WriteLine($" Kurtosis: {Statistics.Kurtosis(logPeaks):F4}"); + +// Jarque-Bera test for normality +double pJB = HypothesisTests.JarqueBeraTest(logPeaks); -// One-sample t-test -double t = HypothesisTests.OneSampleTtest(newSample, mu0); +Console.WriteLine($"\nJarque-Bera Normality Test:"); +Console.WriteLine($" p-value: {pJB:F4}"); -Console.WriteLine($"\nHistorical mean: {mu0:F1}"); -Console.WriteLine($"Current sample mean: {Statistics.Mean(newSample):F2}"); -Console.WriteLine($"t-statistic: {t:F3}"); -Console.WriteLine($"Critical value (two-tailed, α=0.05): ±2.365"); +if (pJB >= 0.05) +{ + Console.WriteLine(" Result: Cannot reject normality of log-transformed data"); + Console.WriteLine(" Implication: Log-Normal distribution may be appropriate"); +} +else +{ + Console.WriteLine(" Result: Log-transformed data departs from normality"); + Console.WriteLine(" Implication: Consider Log-Pearson Type III or GEV distribution"); +} -if (Math.Abs(t) > 2.365) +// Additional check: skewness significance +double skew = Statistics.Skewness(logPeaks); +if (Math.Abs(skew) < 0.5) { - Console.WriteLine("\nProcess has shifted significantly!"); - Console.WriteLine("Action: Investigate and adjust process"); + Console.WriteLine($"\n Note: Skewness ({skew:F3}) is small - Log-Normal may be adequate"); } else { - Console.WriteLine("\nProcess remains in control"); + Console.WriteLine($"\n Note: Skewness ({skew:F3}) is substantial - LP3 recommended"); } ``` -## Interpreting Results +## Test Selection Guide -### p-values -- p < 0.01: Very strong evidence against H₀ -- p < 0.05: Strong evidence against H₀ -- p < 0.10: Weak evidence against H₀ -- p ≥ 0.10: Insufficient evidence to reject H₀ +| Question | Test | Notes | +|----------|------|-------| +| One sample mean vs. hypothesized value | `OneSampleTtest` | Parametric, assumes normality | +| Two independent means (equal variance) | `EqualVarianceTtest` | Use F-test first to check variance equality | +| Two independent means (unequal variance) | `UnequalVarianceTtest` | Welch's t-test, more robust | +| Two paired measurements | `PairedTtest` | Before/after, matched pairs | +| Two variances equal? | `Ftest` | Sensitive to non-normality | +| Model comparison | `FtestModels` | Nested regression models | +| Data normally distributed? | `JarqueBeraTest` | Uses skewness and kurtosis | +| Sequence random? | `WaldWolfowitzTest` | Independence, stationarity | +| Time series autocorrelation? | `LjungBoxTest` | Tests multiple lags | +| Distribution difference? | `MannWhitneyTest` | Non-parametric, rank-based | +| Monotonic trend? | `MannKendallTest` | Non-parametric trend test | +| Linear trend? | `LinearTrendTest` | Parametric trend test | +| Unimodal distribution? | `UnimodalityTest` | GMM-based comparison | + +## Effect Size + +A p-value tells you whether an effect is statistically detectable, but not whether it is practically important. **Effect size** measures the magnitude of a difference independently of sample size. + +**Cohen's d** for two-sample comparisons measures the difference in means in units of standard deviations: + +```math +d = \frac{\bar{x}_1 - \bar{x}_2}{s_p} +``` -### Effect Size -Statistical significance ≠ practical significance. Consider: -- Cohen's d for t-tests: (mean difference) / pooled SD -- Small: d = 0.2, Medium: d = 0.5, Large: d = 0.8 +where $s_p$ is the pooled standard deviation. Conventional thresholds: $|d| < 0.2$ is small, $0.2$–$0.8$ is medium, $> 0.8$ is large. With a large enough sample size, even a tiny $d$ (e.g., 0.01) will produce a significant p-value — but the effect may be meaningless in practice. -### Power -- Probability of detecting true effect -- Influenced by sample size, effect size, α level -- Power ≥ 0.80 typically desired +For trend tests, the **Kendall's tau** correlation (which can be computed from the Mann-Kendall $S$ statistic) serves as a non-parametric effect size measure: $\tau = S / \binom{n}{2}$. ## Best Practices -1. **Check assumptions** before parametric tests (normality, equal variance) -2. **Use non-parametric tests** when assumptions violated -3. **Report effect sizes** along with p-values -4. **Consider multiple testing** corrections if doing many tests -5. **Visualize data** before and after testing -6. **Understand context** - statistical vs practical significance +1. **Choose significance level a priori** — Typically $\alpha = 0.05$, but consider $\alpha = 0.10$ for exploratory analysis +2. **Check test assumptions** — Many tests assume normality or independence +3. **Use non-parametric alternatives** — When assumptions are violated (e.g., Mann-Whitney instead of t-test) +4. **Report p-values, not just decisions** — $p = 0.049$ and $p = 0.051$ are essentially equivalent +5. **Consider multiple testing corrections** — When performing $k$ simultaneous tests, use the Bonferroni correction: reject at $\alpha/k$ instead of $\alpha$ +6. **Report effect sizes** — Statistical significance does not imply practical significance. Always report Cohen's d or an equivalent alongside p-values +7. **Visualize data** — Plots often reveal more than hypothesis tests -## Test Selection Guide +## Important Notes + +- All tests return **p-values**, not test statistics +- Two-sided tests are used by default +- Sample size requirements vary by test (see individual test documentation) +- For hydrological applications, consider the impact of outliers and measurement uncertainty + +--- + +## References + +[1] Helsel, D. R., Hirsch, R. M., Ryberg, K. R., Archfield, S. A., & Gilroy, E. J. (2020). *Statistical Methods in Water Resources*. U.S. Geological Survey Techniques and Methods, Book 4, Chapter A3. -| Question | Test | -|----------|------| -| One sample mean vs. value | One-sample t-test | -| Two independent means | Two-sample t-test (or Mann-Whitney) | -| Two paired measurements | Paired t-test | -| Two variances | F-test | -| Normality | Jarque-Bera | -| Trend existence | Mann-Kendall | -| Linear relationship | Linear trend test | -| Autocorrelation | Ljung-Box | -| Randomness | Wald-Wolfowitz | +[2] Hirsch, R. M., Slack, J. R., & Smith, R. A. (1982). Techniques of trend analysis for monthly water quality data. *Water Resources Research*, 18(1), 107-121. --- -[← Previous: Goodness-of-Fit](goodness-of-fit.md) | [Back to Index](../index.md) | [Next: Convergence Diagnostics →](../sampling/convergence-diagnostics.md) +[← Previous: Goodness-of-Fit](goodness-of-fit.md) | [Back to Index](../index.md) | [Next: Univariate Distributions →](../distributions/univariate.md) diff --git a/paper/paper.bib b/paper/paper.bib new file mode 100644 index 00000000..b3dcfaca --- /dev/null +++ b/paper/paper.bib @@ -0,0 +1,305 @@ +@techreport{england2018, + author = {England, John F. and Cohn, Timothy A. and Faber, Beth A. and Stedinger, Jery R. and Thomas, Wilbert O. and Veilleux, Andrea G. and Kiang, Julie E. and Mason, Robert R.}, + title = {Guidelines for Determining Flood Flow Frequency---{Bulletin} 17C}, + institution = {U.S. Geological Survey}, + type = {Techniques and Methods}, + number = {Book 4, Chapter B5}, + year = {2018}, + pages = {148}, + doi = {10.3133/tm4B5} +} + +@article{cohn2013, + author = {Cohn, Timothy A. and England, John F. and Berenbrock, Charles E. and Mason, Robert R. and Stedinger, Jery R. and Lamontagne, Jonathan R.}, + title = {A generalized {Grubbs-Beck} test statistic for detecting multiple potentially influential low outliers in flood series}, + journal = {Water Resources Research}, + volume = {49}, + number = {8}, + pages = {5047--5058}, + year = {2013}, + doi = {10.1002/wrcr.20392} +} + +@article{harris2020, + author = {Harris, Charles R. and Millman, K. Jarrod and van der Walt, St\'{e}fan J. and Gommers, Ralf and Virtanen, Pauli and Cournapeau, David and Wieser, Eric and Taylor, Julian and Berg, Sebastian and Smith, Nathaniel J. and others}, + title = {Array programming with {NumPy}}, + journal = {Nature}, + volume = {585}, + pages = {357--362}, + year = {2020}, + doi = {10.1038/s41586-020-2649-2} +} + +@article{virtanen2020, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and Haberland, Matt and Reddy, Tyler and Cournapeau, David and Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and Bright, Jonathan and others}, + title = {{SciPy} 1.0: Fundamental algorithms for scientific computing in {Python}}, + journal = {Nature Methods}, + volume = {17}, + pages = {261--272}, + year = {2020}, + doi = {10.1038/s41592-019-0686-2} +} + +@article{hosking1990, + author = {Hosking, J. R. M.}, + title = {{L}-moments: Analysis and estimation of distributions using linear combinations of order statistics}, + journal = {Journal of the Royal Statistical Society: Series B (Methodological)}, + volume = {52}, + number = {1}, + pages = {105--124}, + year = {1990}, + doi = {10.1111/j.2517-6161.1990.tb01775.x} +} + +@book{hosking1997, + author = {Hosking, J. R. M. and Wallis, James R.}, + title = {Regional Frequency Analysis: An Approach Based on L-Moments}, + publisher = {Cambridge University Press}, + year = {1997}, + doi = {10.1017/CBO9780511529443} +} + +@article{haario2001, + author = {Haario, Heikki and Saksman, Eero and Tamminen, Johanna}, + title = {An adaptive {Metropolis} algorithm}, + journal = {Bernoulli}, + volume = {7}, + number = {2}, + pages = {223--242}, + year = {2001}, + doi = {10.2307/3318737} +} + +@article{terbraak2008, + author = {ter Braak, Cajo J. F. and Vrugt, Jasper A.}, + title = {Differential Evolution {Markov Chain} with snooker updater and fewer chains}, + journal = {Statistics and Computing}, + volume = {18}, + number = {4}, + pages = {435--446}, + year = {2008}, + doi = {10.1007/s11222-008-9104-9} +} + +@incollection{neal2011, + author = {Neal, Radford M.}, + title = {{MCMC} using {Hamiltonian} dynamics}, + booktitle = {Handbook of Markov Chain Monte Carlo}, + editor = {Brooks, Steve and Gelman, Andrew and Jones, Galin L. and Meng, Xiao-Li}, + publisher = {CRC Press}, + pages = {113--162}, + year = {2011}, + chapter = {5} +} + +@article{vehtari2021, + author = {Vehtari, Aki and Gelman, Andrew and Simpson, Daniel and Carpenter, Bob and B\"{u}rkner, Paul-Christian}, + title = {Rank-normalization, folding, and localization: An improved {R-hat} for assessing convergence of {MCMC}}, + journal = {Bayesian Analysis}, + volume = {16}, + number = {2}, + pages = {667--718}, + year = {2021}, + doi = {10.1214/20-BA1221} +} + +@book{efron1993, + author = {Efron, Bradley and Tibshirani, Robert J.}, + title = {An Introduction to the Bootstrap}, + publisher = {Chapman \& Hall}, + year = {1993}, + isbn = {978-0412042317} +} + +@article{storn1997, + author = {Storn, Rainer and Price, Kenneth}, + title = {Differential evolution -- a simple and efficient heuristic for global optimization over continuous spaces}, + journal = {Journal of Global Optimization}, + volume = {11}, + number = {4}, + pages = {341--359}, + year = {1997}, + doi = {10.1023/A:1008202821328} +} + +@article{duan1994, + author = {Duan, Qingyun and Sorooshian, Soroosh and Gupta, Vijai K.}, + title = {Optimal use of the {SCE-UA} global optimization method for calibrating watershed models}, + journal = {Journal of Hydrology}, + volume = {158}, + number = {3--4}, + pages = {265--284}, + year = {1994}, + doi = {10.1016/0022-1694(94)90057-4} +} + +@inproceedings{kennedy1995, + author = {Kennedy, James and Eberhart, Russell}, + title = {Particle swarm optimization}, + booktitle = {Proceedings of ICNN'95 - International Conference on Neural Networks}, + volume = {4}, + pages = {1942--1948}, + year = {1995}, + doi = {10.1109/ICNN.1995.488968} +} + +@article{nelder1965, + author = {Nelder, J. A. and Mead, R.}, + title = {A simplex method for function minimization}, + journal = {The Computer Journal}, + volume = {7}, + number = {4}, + pages = {308--313}, + year = {1965}, + doi = {10.1093/comjnl/7.4.308} +} + +@misc{mathnetnumerics, + author = {Ruegg, Christoph and Cuda, Marcus and Van Gael, Jurgen}, + title = {{Math.NET} Numerics}, + year = {2024}, + url = {https://numerics.mathdotnet.com/}, + note = {Version 5.0} +} + +@manual{hosking2019lmom, + author = {Hosking, J. R. M.}, + title = {lmom: {L}-moments}, + year = {2019}, + note = {R package version 2.8}, + url = {https://CRAN.R-project.org/package=lmom} +} + +@article{stephenson2002evd, + author = {Stephenson, Alec G.}, + title = {{evd}: Extreme Value Distributions}, + journal = {R News}, + volume = {2}, + number = {2}, + pages = {31--32}, + year = {2002}, + url = {https://CRAN.R-project.org/package=evd} +} + +@book{press2007, + author = {Press, William H. and Teukolsky, Saul A. and Vetterling, William T. and Flannery, Brian P.}, + title = {Numerical Recipes: The Art of Scientific Computing}, + edition = {3rd}, + publisher = {Cambridge University Press}, + year = {2007}, + isbn = {978-0521880688} +} + +@book{rao2000, + author = {Rao, A. Ramachandra and Hamed, Khaled H.}, + title = {Flood Frequency Analysis}, + publisher = {CRC Press}, + year = {2000}, + isbn = {978-0849300837} +} + +@manual{viglione2024nsrfa, + author = {Viglione, Alberto}, + title = {nsRFA: Non-supervised Regional Frequency Analysis}, + year = {2024}, + note = {R package version 0.7-17}, + url = {https://CRAN.R-project.org/package=nsRFA} +} + +@manual{ribatet2022spatialextremes, + author = {Ribatet, Mathieu}, + title = {SpatialExtremes: Modelling Spatial Extremes}, + year = {2022}, + note = {R package version 2.1-0}, + url = {https://CRAN.R-project.org/package=SpatialExtremes} +} + +@article{rinnooy1987, + author = {{Rinnooy Kan}, A. H. G. and Timmer, G. T.}, + title = {Stochastic global optimization methods part {II}: Multi level methods}, + journal = {Mathematical Programming}, + volume = {39}, + number = {1}, + pages = {57--78}, + year = {1987}, + doi = {10.1007/BF02592071} +} + +@article{hoffman2014, + author = {Hoffman, Matthew D. and Gelman, Andrew}, + title = {The {No-U-Turn} Sampler: Adaptively setting path lengths in {Hamiltonian Monte Carlo}}, + journal = {Journal of Machine Learning Research}, + volume = {15}, + number = {47}, + pages = {1593--1623}, + year = {2014}, + url = {https://jmlr.org/papers/v15/hoffman14a.html} +} + +@article{gelmanrubin1992, + author = {Gelman, Andrew and Rubin, Donald B.}, + title = {Inference from Iterative Simulation Using Multiple Sequences}, + journal = {Statistical Science}, + volume = {7}, + number = {4}, + pages = {457--472}, + year = {1992}, + doi = {10.1214/ss/1177011136} +} + +@article{waylen1982, + author = {Waylen, Peter and Woo, Ming-Ko}, + title = {Prediction of annual floods generated by mixed processes}, + journal = {Water Resources Research}, + volume = {18}, + number = {4}, + pages = {1283--1286}, + year = {1982}, + doi = {10.1029/WR018i004p01283} +} + +@article{alila2002, + author = {Alila, Younes and Mtiraoui, Ahmed}, + title = {Implications of heterogeneous flood-frequency distributions on traditional stream-discharge prediction techniques}, + journal = {Hydrological Processes}, + volume = {16}, + number = {5}, + pages = {1065--1084}, + year = {2002}, + doi = {10.1002/hyp.346} +} + +@book{crowder2001, + author = {Crowder, Martin J.}, + title = {Classical Competing Risks}, + publisher = {Chapman and Hall/CRC}, + year = {2001}, + doi = {10.1201/9781420035902} +} + +@book{bedford2001, + author = {Bedford, Tim and Cooke, Roger}, + title = {Probabilistic Risk Analysis: Foundations and Methods}, + publisher = {Cambridge University Press}, + year = {2001}, + doi = {10.1017/CBO9780511813597} +} + +@book{piessens1983, + author = {Piessens, Robert and de Doncker-Kapenga, Elise and \"{U}berhuber, Christoph W. and Kahaner, David K.}, + title = {{QUADPACK}: A Subroutine Package for Automatic Integration}, + publisher = {Springer-Verlag}, + year = {1983}, + doi = {10.1007/978-3-642-61786-7} +} + +@article{lepage1978, + author = {Lepage, G. Peter}, + title = {A new algorithm for adaptive multidimensional integration}, + journal = {Journal of Computational Physics}, + volume = {27}, + number = {2}, + pages = {192--203}, + year = {1978}, + doi = {10.1016/0021-9991(78)90004-9} +} diff --git a/paper/paper.md b/paper/paper.md new file mode 100644 index 00000000..e47fa508 --- /dev/null +++ b/paper/paper.md @@ -0,0 +1,130 @@ +--- +title: 'Numerics: A .NET Library for Numerical Computing, Statistical Analysis, and Risk Assessment' +tags: + - C# + - .NET + - numerical methods + - statistics + - probability distributions + - Bayesian inference + - machine learning + - optimization +authors: + - name: C. Haden Smith + orcid: 0000-0002-4651-9890 + affiliation: 1 + corresponding: true + - name: Woodrow L. Fields + affiliation: 1 + orcid: 0009-0008-7454-5552 + - name: Julian Gonzalez + affiliation: 1 + orcid: 0009-0009-9058-7653 + - name: Sadie Niblett + affiliation: 1 + orcid: 0009-0008-8588-4816 + - name: Brennan Beam + affiliation: 2 + orcid: 0009-0003-0515-3727 + - name: Brian Skahill + orcid: 0000-0002-2164-0301 + affiliation: 3 +affiliations: + - name: U.S. Army Corps of Engineers, Risk Management Center, Lakewood, Colorado, USA + index: 1 + - name: U.S. Army Corps of Engineers, Hydrologic Engineering Center, Davis, California, USA + index: 2 + - name: Fariborz Maseeh Department of Mathematics and Statistics, Portland State University, Portland, Oregon, USA + index: 3 +date: 8 March 2026 +bibliography: paper.bib +--- + +# Summary + +Numerics is a free and open-source numerical computing library for .NET, developed by the U.S. Army Corps of Engineers Risk Management Center (USACE-RMC). The library provides 43 univariate probability distributions with L-moment and maximum likelihood parameter estimation, eight Markov Chain Monte Carlo (MCMC) samplers for Bayesian inference, bootstrap uncertainty quantification, bivariate copulas, optimization algorithms, and numerical methods for integration, differentiation, root finding, and linear algebra. Numerics targets engineers, scientists, and analysts working in infrastructure risk assessment, flood frequency analysis, and Monte Carlo simulation within .NET-based enterprise systems. The library supports .NET 8.0 through 10.0 and .NET Framework 4.8.1 and is validated by over 1,000 unit tests against published reference values. + +# Statement of Need + +Infrastructure risk assessment for dams and levees requires the integration of extreme value statistics, uncertainty quantification, and Monte Carlo simulation. Practitioners face challenges including small sample sizes from historical records, regulatory requirements for specific statistical methods such as the Log-Pearson Type III distribution mandated by USGS Bulletin 17C [@england2018], robust outlier handling via the Multiple Grubbs-Beck Test [@cohn2013], and the need to incorporate expert judgment through Bayesian inference. + +The .NET framework is a widely used platform for enterprise and government desktop applications in the United States, yet the ecosystem lacks comprehensive numerical libraries tailored for engineering risk analysis. Python offers mature tools such as NumPy [@harris2020] and SciPy [@virtanen2020], and R provides specialized hydrology packages such as `lmom` [@hosking2019lmom], `nsRFA` [@viglione2024nsrfa], and `SpatialExtremes` [@ribatet2022spatialextremes], but neither ecosystem integrates with .NET-based engineering workflows without introducing language interop complexity, performance overhead, and deployment challenges for regulated government systems. In addition, Numerics benefits from .NET's ahead-of-time compilation, absence of a global interpreter lock, and native `Parallel.For` support, delivering significantly higher throughput for computationally intensive methods such as bootstrap resampling and MCMC sampling compared to interpreted alternatives. + +Numerics fills this gap by providing domain-specific capabilities within .NET: + +- **Hydrology-specific distributions**: Log-Pearson Type III, Generalized Extreme Value, Pearson Type III, Generalized Pareto, and Kappa-Four, with L-moment parameter estimation [@hosking1990; @hosking1997] that outperforms conventional moments for the small samples typical of flood records. +- **Mixed-population and competing-risk models**: Mixture distributions [@waylen1982; @alila2002] for sites where floods arise from distinct causal mechanisms such as rainfall, snowmelt, or dam-regulated releases, and competing-risks models [@crowder2001; @bedford2001] for reliability analysis with multiple failure modes. +- **Bayesian inference**: Eight MCMC samplers, including Random Walk Metropolis-Hastings, Adaptive RWMH [@haario2001], Differential Evolution MCMC (DE-MCz and DE-MCzs) [@terbraak2008], Hamiltonian Monte Carlo [@neal2011], No-U-Turn Sampler (NUTS) [@hoffman2014], Gibbs, and Self-Normalizing Importance Sampling (SNIS), with improved Gelman-Rubin convergence diagnostics [@gelmanrubin1992; @vehtari2021]. +- **Uncertainty quantification**: Bootstrap resampling methods [@efron1993] for confidence intervals on design estimates, a requirement in dam and levee safety risk assessments. +- **Global optimization**: Differential Evolution [@storn1997], Shuffled Complex Evolution [@duan1994], Particle Swarm Optimization [@kennedy1995], Multi-Level Single-Linkage (MLSL) [@rinnooy1987], and Nelder-Mead [@nelder1965] for calibrating complex, multi-modal objective functions. +- **Adaptive numerical integration**: Gauss-Kronrod quadrature [@piessens1983] and VEGAS adaptive Monte Carlo integration [@lepage1978] for efficiently evaluating complex risk integrals involving multiple system components and failure modes, a core computation in quantitative risk assessment. +- **Machine learning**: Supervised and unsupervised algorithms, including generalized linear models, decision trees, random forests, k-nearest neighbors, k-means clustering, and Gaussian mixture models for regression, classification, and clustering tasks in risk assessment workflows. + +# State of the Field + +General-purpose .NET numerical libraries exist, most notably Math.NET Numerics [@mathnetnumerics], which provides linear algebra, probability distributions, and basic statistics. However, Math.NET Numerics does not include L-moment estimation, hydrology-specific distributions (Log-Pearson Type III, Kappa-Four), mixture and competing-risk models, adaptive integration, bootstrap resampling, copulas, or global optimization algorithms. Math.NET Numerics includes basic MCMC samplers (Metropolis-Hastings, Hybrid Monte Carlo, Slice), but lacks the adaptive and ensemble methods (Adaptive RWMH, DE-MCzs, NUTS) commonly applied for Bayesian inference of hydrologic models. Contributing these features to Math.NET Numerics was not pursued because the scope of domain-specific functionality, the distinct API design requirements for risk assessment workflows, and the need for long-term maintenance by a domain-expert organization warranted an independent library. + +In other language ecosystems, Python's SciPy [@virtanen2020] provides broad numerical capabilities but lacks specialized hydrological distributions and L-moment estimation. R packages such as `lmom` [@hosking2019lmom] and `evd` [@stephenson2002evd] offer these features individually, but integrating R into .NET production applications introduces runtime dependencies and complicates deployment in regulated government environments. No single package in any ecosystem consolidates mixture distributions, competing-risk models, adaptive integration, extreme-value statistics, global optimization, bootstrapping, machine learning, and MCMC into a unified library for engineering risk analysis. + +Numerics consolidates these capabilities into a single, self-contained .NET library purpose-built for quantitative risk assessment in water resources engineering. Computationally intensive operations, including bootstrap resampling and MCMC chain evaluation, are parallelized using `Parallel.For` for high-throughput execution on modern multi-core hardware. Backed by long-term USACE-RMC maintenance, Numerics provides the ecosystem, performance, and institutional support needed for quantitative risk analysis in dam and levee safety. + +# Research Impact + +Numerics serves as the computational engine for six USACE-RMC production applications used for infrastructure safety decisions: + +- **[RMC-BestFit](https://github.com/USACE-RMC/RMC-BestFit)**: Bayesian estimation and fitting software that supports flood frequency analysis for flood risk management and dam and levee safety studies. +- **[RMC-RFA](https://github.com/USACE-RMC/RMC-RFA)**: Reservoir frequency analysis software to facilitate flood hazard assessments for dam safety studies. +- **[RMC-TotalRisk](https://github.com/USACE-RMC/RMC-TotalRisk)**: Quantitative risk analysis software designed to support dam and levee safety investment decisions. +- **[LifeSim](https://github.com/USACE-RMC/LifeSim)**: A consequence estimation engine used to simulate population redistribution during an evacuation. +- **[Levee Screening Tool](https://lst2.sec.usace.army.mil/)**: A web-based portfolio-level levee risk screening tool that has been used to evaluate more than 7,500 levee segments across the national inventory. +- **[Dam Screening Tool](https://dst.sec.usace.army.mil/)**: A web-based portfolio-level dam risk screening tool that will be used to evaluate more than 90,000 dams across the national inventory. + +These applications support risk-informed decisions for thousands of dams and levees managed by USACE, other federal agencies, and private dam and levee owners, directly affecting public safety for millions of Americans. The RMC software suite is also used internationally by organizations in Australia, the Netherlands, and Canada. By open-sourcing Numerics, USACE-RMC enables independent verification of the algorithms underpinning these critical assessments and invites contributions from the broader water resources engineering community. + +# Software Design + +The library is organized around abstract base classes that define consistent interfaces for each computational domain. `UnivariateDistributionBase` provides PDF, CDF, inverse CDF, log-likelihood evaluation, and random variate generation for all 43 univariate distributions. Parameter estimation is decoupled from distribution classes through segregated interfaces (`IMaximumLikelihoodEstimation`, `ILinearMomentEstimation`, `IMomentEstimation`), so each distribution declares which estimation methods it supports and analysts can swap candidate distributions without changing workflow code. `Optimizer` provides a common minimization interface shared by all local and global optimization algorithms, with built-in function evaluation tracking, convergence testing, and numerical Hessian computation. `MCMCSampler` manages chain initialization, parallel execution via `Parallel.For`, thinning, and posterior output collection for all eight samplers, while convergence diagnostics are handled by a separate `MCMCDiagnostics` utility class. Bivariate copulas follow the same pattern through `BivariateCopula` and its `ArchimedeanCopula` specialization. + +The library has zero external runtime dependencies, relying only on .NET Base Class Libraries, which eliminates NuGet dependency conflicts in large government applications and simplifies deployment in regulated environments. Multi-targeting (.NET 8.0, 9.0, 10.0, and .NET Framework 4.8.1) ensures compatibility with both modern and legacy enterprise systems. Because .NET 8 and later are cross-platform, Numerics runs on Windows, Linux, and macOS, supporting both desktop engineering tools and cloud-hosted web applications. Python users can also access Numerics through PythonNet, making the library's performance and capabilities available to the broader scientific and academic community. + +Continuous integration via GitHub Actions runs the test suite (over 1,000 tests validated against published references [@press2007; @rao2000; @england2018]) on every pull request, preventing regressions in numerical accuracy. + +# Example + +The following example illustrates a bootstrap uncertainty analysis [@efron1993] for flood frequency quantiles, a common workflow in dam and levee safety studies. A Log-Pearson Type III distribution is fit to a synthetic sample using L-moments, and bias-corrected and accelerated (BCa) confidence intervals are computed for a range of non-exceedance probabilities: + +```csharp +using Numerics.Distributions; + +// Generate a synthetic flood record from a Log-Pearson Type III distribution +int n = 80; +int B = 10000; +var lp3 = new LogPearsonTypeIII(meanOfLog: 3, standardDeviationOfLog: 0.5, skewOfLog: 0.25); +double[] sample = lp3.GenerateRandomValues(n, seed: 12345); + +// Estimate distribution parameters from the sample using L-moments +lp3.Estimate(sample, ParameterEstimationMethod.MethodOfLinearMoments); + +// Configure the bootstrap analysis +var boot = new BootstrapAnalysis(distribution: lp3, + estimationMethod: ParameterEstimationMethod.MethodOfLinearMoments, + sampleSize: n, + replications: B, + seed: 12345); + +// Define non-exceedance probabilities for quantile estimation +var probabilities = new double[] { 0.999, 0.99, 0.9, 0.8, 0.7, 0.5, 0.3, 0.2, 0.1, 0.05, 0.02, 0.01 }; + +// Compute 90% BCa confidence intervals for each quantile +var CIs = boot.BCaQuantileCI(sample, probabilities, alpha: 0.1); +``` + +# AI Usage Disclosure + +Generative AI was used to assist with XML documentation comments, markdown documentation content, and code review during library development. All core library code, class architecture, numerical methods, and algorithms were designed and implemented by the authors. All AI-generated content was reviewed, edited, and validated by the authors, who made all design decisions and accept full responsibility for the work. + +# Acknowledgements + +The authors acknowledge the U.S. Army Corps of Engineers Risk Management Center for supporting the development and open-source release of this software. The Numerics software would not exist without the support of Risk Management Center leadership, in particular the RMC Director Bryant A. Robbins and RMC Lead Engineers David A. Margo (retired) and John F. England. The authors are grateful to all who have contributed to the development of Numerics and the content of this paper. + +# References